diff --git a/cmake/libraries.cmake b/cmake/libraries.cmake index 5303497394c..aadb783cc41 100644 --- a/cmake/libraries.cmake +++ b/cmake/libraries.cmake @@ -22,7 +22,7 @@ set(FLB_PATH_LIB_MINIZ "lib/miniz") set(FLB_PATH_LIB_TUTF8E "lib/tutf8e") set(FLB_PATH_LIB_CARES "lib/c-ares-1.34.6") set(FLB_PATH_LIB_SNAPPY "lib/snappy-fef67ac") -set(FLB_PATH_LIB_RDKAFKA "lib/librdkafka-2.10.1") +set(FLB_PATH_LIB_RDKAFKA "lib/librdkafka-2.15.0") set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb") set(FLB_PATH_LIB_WASM_MICRO_RUNTIME "lib/wasm-micro-runtime-WAMR-2.4.1") set(FLB_PATH_LIB_ZSTD "lib/zstd-1.5.7") diff --git a/lib/librdkafka-2.10.1/.semaphore/semaphore.yml b/lib/librdkafka-2.10.1/.semaphore/semaphore.yml deleted file mode 100644 index cd6c1739447..00000000000 --- a/lib/librdkafka-2.10.1/.semaphore/semaphore.yml +++ /dev/null @@ -1,404 +0,0 @@ -version: v1.0 -name: 'librdkafka build and release artifact pipeline' -agent: - machine: - type: s1-prod-macos-13-5-arm64 -execution_time_limit: - hours: 3 -global_job_config: - prologue: - commands: - - checkout - - mkdir artifacts - - mkdir dest -blocks: - - name: 'OSX arm64/m1' - dependencies: [] - task: - agent: - machine: - type: s1-prod-macos-13-5-arm64 - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-osx__arch-arm64__lnk-all - epilogue: - commands: - - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' - jobs: - - name: 'Build' - commands: - - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip - - make -j all examples check - - examples/rdkafka_example -X builtin.features - - otool -L src/librdkafka.dylib - - otool -L src-cpp/librdkafka++.dylib - - make -j -C tests build - - make -C tests run_local_quick - - DESTDIR="$PWD/dest" make install - - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .) - - - - name: 'OSX x64' - dependencies: [] - task: - agent: - machine: - type: s1-prod-macos-13-5-amd64 - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-osx__arch-x64__lnk-all - epilogue: - commands: - - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' - jobs: - - name: 'Build' - commands: - - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip - - make -j all examples check - - examples/rdkafka_example -X builtin.features - - otool -L src/librdkafka.dylib - - otool -L src-cpp/librdkafka++.dylib - - make -j -C tests build - - make -C tests run_local_quick - - DESTDIR="$PWD/dest" make install - - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .) - - - - name: 'Style check' - dependencies: [] - skip: - # Skip for release tags, we don't want style checks - # to fail the release build. - when: "tag =~ '^v[0-9]\\.'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: 'Style check' - commands: - - sudo apt update - - sudo apt install -y clang-format-18 python3 python3-pip python3-setuptools - - python3 -m pip install -r packaging/tools/requirements.txt - - make style-check - - - - name: 'Build documentation' - dependencies: [] - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-1 - jobs: - - name: 'Generate documentation' - commands: - - NEEDRESTART_MODE=a sudo -E apt install -y doxygen graphviz - - make docs - - (cd staging-docs && tar cvzf ../artifacts/librdkafka-docs.tgz .) - - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/librdkafka-docs.tgz --destination artifacts/librdkafka-docs.tgz' - - - - name: 'Linux Ubuntu x64: source build' - dependencies: [] - skip: - # Skip for release tags, we don't want flaky CI tests - # to fail the release build. - when: "tag =~ '^v[0-9]\\.'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-2 - env_vars: - - name: CFLAGS - value: -std=gnu90 # Test minimum C standard, default in CentOS 7 - prologue: - commands: - - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' - - sem-version java 17 - jobs: - - name: 'Build configuration checks' - commands: - - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb - - sudo dpkg -i rapidjson-dev.deb - - python3 -m pip install -U pip - - ./packaging/tools/build-configurations-checks.sh - - name: 'Build and integration tests with "classic" protocol' - commands: - - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb - - sudo dpkg -i rapidjson-dev.deb - - python3 -m pip install -U pip - - python3 -m pip -V - - (cd tests && python3 -m pip install -r requirements.txt) - - ./configure --install-deps --enable-werror --enable-devel - # split these up - - ./packaging/tools/rdutcoverage.sh - - make copyright-check - - make -j all examples check - - echo "Verifying that CONFIGURATION.md does not have manual changes" - - git diff --exit-code CONFIGURATION.md - - examples/rdkafka_example -X builtin.features - - ldd src/librdkafka.so.1 - - ldd src-cpp/librdkafka++.so.1 - - make -j -C tests build - - make -C tests run_local_quick - - DESTDIR="$PWD/dest" make install - - (cd tests && python3 -m trivup.clusters.KafkaCluster --version 3.9.0 --cpversion 7.9.0 --cmd 'make quick') - - name: 'Build and integration tests with "consumer" protocol' - commands: - - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb - - sudo dpkg -i rapidjson-dev.deb - - python3 -m pip install -U pip - - python3 -m pip -V - - (cd tests && python3 -m pip install -r requirements.txt) - - ./configure --install-deps --enable-werror --enable-devel - # split these up - - ./packaging/tools/rdutcoverage.sh - - make copyright-check - - make -j all examples check - - echo "Verifying that CONFIGURATION.md does not have manual changes" - - git diff --exit-code CONFIGURATION.md - - examples/rdkafka_example -X builtin.features - - ldd src/librdkafka.so.1 - - ldd src-cpp/librdkafka++.so.1 - - make -j -C tests build - - make -C tests run_local_quick - - DESTDIR="$PWD/dest" make install - - export TEST_KAFKA_VERSION=4.0.0 - - export TEST_CONSUMER_GROUP_PROTOCOL=consumer - - (cd tests && python3 -m trivup.clusters.KafkaCluster --kraft --version '4.0.0' - --cpversion 7.9.0 - --cmd 'make quick') - - - - name: 'Linux x64: release artifact docker builds' - dependencies: [] - run: - when: "tag =~ '^v[0-9]\\.'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-2 - prologue: - commands: - - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' - epilogue: - commands: - - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' - jobs: - - name: 'Build: centos8 glibc +gssapi' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi - commands: - - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz - - - name: 'Build: centos8 glibc' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all - commands: - - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz - - - name: 'Build: alpine musl +gssapi' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi - commands: - - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz - - - name: 'Build: alpine musl' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all - commands: - - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz - - - - name: 'Linux arm64: release artifact docker builds' - dependencies: [] - task: - agent: - machine: - type: s1-prod-ubuntu24-04-arm64-2 - prologue: - commands: - - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' - epilogue: - commands: - - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' - jobs: - - name: 'Build: centos8 glibc +gssapi' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi - commands: - - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz - - - name: 'Build: centos8 glibc' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all - commands: - - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz - - - name: 'Build: alpine musl +gssapi' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi - commands: - - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz - - - name: 'Build: alpine musl' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all - commands: - - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz - - - - name: 'Windows x64: MinGW-w64' - dependencies: [] - task: - agent: - machine: - type: s1-prod-windows - env_vars: - - name: CHERE_INVOKING - value: 'yes' - - name: MSYSTEM - value: UCRT64 - prologue: - commands: - # Set up msys2 - - "& .\\win32\\setup-msys2.ps1" - epilogue: - commands: - - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ } - jobs: - - name: 'Build: MinGW-w64 Dynamic' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std - commands: - - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh ./artifacts/librdkafka.tgz' - - - name: 'Build: MinGW-w64 Static' - env_vars: - - name: ARTIFACT_KEY - value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static - commands: - - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh --static ./artifacts/librdkafka.tgz' - - - name: 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019' - dependencies: [] - task: - agent: - machine: - type: s1-prod-windows - env_vars: - # Disable vcpkg telemetry - - name: VCPKG_DISABLE_METRICS - value: 'yes' - prologue: - commands: - # install vcpkg in the parent directory. - - cd .. - # Setup vcpkg - - "& .\\librdkafka\\win32\\setup-vcpkg.ps1" - - cd librdkafka - - ..\vcpkg\vcpkg integrate install - # Install required packages. - - ..\vcpkg\vcpkg --feature-flags=versions install --triplet $Env:triplet - epilogue: - commands: - - Get-ChildItem . -include *.dll -recurse - - Get-ChildItem . -include *.lib -recurse - - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ } - jobs: - - name: 'Build: MSVC x64' - env_vars: - - name: triplet - value: x64-windows - - name: ARTIFACT_KEY - value: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std - commands: - - "& .\\win32\\msbuild.ps1 -platform x64" - - "& .\\win32\\package-zip.ps1 -platform x64" - - name: 'Build: MSVC x86' - env_vars: - - name: triplet - value: x86-windows - - name: ARTIFACT_KEY - value: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std - commands: - - "& .\\win32\\msbuild.ps1 -platform Win32" - - "& .\\win32\\package-zip.ps1 -platform Win32" - - - name: 'Packaging' - dependencies: - - 'Build documentation' - - 'OSX arm64/m1' - - 'OSX x64' - - 'Linux x64: release artifact docker builds' - - 'Linux arm64: release artifact docker builds' - - 'Windows x64: MinGW-w64' - - 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019' - run: - when: "tag =~ '^v[0-9]\\.'" - task: - agent: - machine: - type: s1-prod-ubuntu24-04-amd64-2 - jobs: - - name: 'Build NuGet and static packages' - commands: - # Get all artifacts from previous jobs in this workflow/pipeline. - - artifact pull workflow artifacts - - mkdir -p packages - # Prepare packaging tools - - cd packaging/nuget - - python3 -m pip install -U -r requirements.txt - # Create NuGet package - # We need --ignore-tag since the jobs don't add the tag to - # the artifact path, and they don't need to since these artifacts - # are part of the same workflow. - - ./release.py --directory ../../artifacts --ignore-tag --class NugetPackage ${SEMAPHORE_GIT_TAG_NAME} - - cp -v librdkafka.redist.*.nupkg ../../packages - # Create static package - - ./release.py --directory ../../artifacts --ignore-tag --class StaticPackage ${SEMAPHORE_GIT_TAG_NAME} - - cp -v librdkafka-static-bundle*.tgz ../../packages - - cd ../../ - # Copy generated docs to packages for inclusion in the tar ball - - cp -v artifacts/librdkafka-docs.tgz packages/ - # Maker super tar ball of all packages - - cd packages - - tar cvf librdkafka-packages-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tar . - # Provide some extra details - - ls -la - - sha256sum * - - cd .. - # Upload all packages to project artifact store - - artifact push project packages --destination librdkafka-packages-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID} - - echo Thank you - -promotions: - - name: Run all tests on master commits - pipeline_file: run-all-tests.yml - parameters: - env_vars: - - required: true - name: TEST_KAFKA_GIT_REF - default_value: 3.8.0 - - required: true - name: TEST_TYPE - default_value: plaintext,ssl - - required: true - name: TEST_ARCHES - default_value: x86_64,aarch64 - - required: true - name: TEST_PARALLEL - default_value: "1" - auto_promote_on: - - result: passed - branch: - - "master" diff --git a/lib/librdkafka-2.10.1/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch b/lib/librdkafka-2.10.1/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch deleted file mode 100644 index 6623b22fbb3..00000000000 --- a/lib/librdkafka-2.10.1/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/configure 2022-06-27 12:15:45.000000000 +0200 -+++ b/configure 2022-06-27 12:17:20.000000000 +0200 -@@ -33432,7 +33432,7 @@ - - - -- if test "x$cross_compiling" != xyes; then -+ if false; then - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking run-time libs availability" >&5 - printf %s "checking run-time libs availability... " >&6; } diff --git a/lib/librdkafka-2.10.1/packaging/mingw-w64/run-tests.sh b/lib/librdkafka-2.10.1/packaging/mingw-w64/run-tests.sh deleted file mode 100644 index 6749add5d42..00000000000 --- a/lib/librdkafka-2.10.1/packaging/mingw-w64/run-tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -cd tests -./test-runner.exe -l -Q -p1 0000 diff --git a/lib/librdkafka-2.10.1/packaging/tools/build-configurations-checks.sh b/lib/librdkafka-2.10.1/packaging/tools/build-configurations-checks.sh deleted file mode 100755 index 6001fa9cc80..00000000000 --- a/lib/librdkafka-2.10.1/packaging/tools/build-configurations-checks.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -# Disable all flags to make sure it -# compiles correctly in all cases -./configure --install-deps --disable-ssl --disable-gssapi \ ---disable-curl --disable-zlib \ ---disable-zstd --disable-lz4-ext --disable-regex-ext \ ---disable-c11threads --disable-syslog \ ---enable-werror --enable-devel -make -j -make -j -C tests run_local_quick diff --git a/lib/librdkafka-2.10.1/src/cJSON.c b/lib/librdkafka-2.10.1/src/cJSON.c deleted file mode 100644 index 9aec18469c1..00000000000 --- a/lib/librdkafka-2.10.1/src/cJSON.c +++ /dev/null @@ -1,2834 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -/* cJSON */ -/* JSON parser in C. */ - -/* disable warnings about old C89 functions in MSVC */ -#if !defined(_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) -#define _CRT_SECURE_NO_DEPRECATE -#endif - -#ifdef __GNUC__ -#pragma GCC visibility push(default) -#endif -#if defined(_MSC_VER) -#pragma warning(push) -/* disable warning about single line comments in system headers */ -#pragma warning(disable : 4001) -#endif - -#include -#include -#include -#include -#include -#include -#include - -#ifdef ENABLE_LOCALES -#include -#endif - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif -#ifdef __GNUC__ -#pragma GCC visibility pop -#endif - -#include "cJSON.h" - -/* define our own boolean type */ -#ifdef true -#undef true -#endif -#define true ((cJSON_bool)1) - -#ifdef false -#undef false -#endif -#define false ((cJSON_bool)0) - -/* define isnan and isinf for ANSI C, if in C99 or above, isnan and isinf has - * been defined in math.h */ -#ifndef isinf -#define isinf(d) (isnan((d - d)) && !isnan(d)) -#endif -#ifndef isnan -#define isnan(d) (d != d) -#endif - -#ifndef NAN -#define NAN 0.0 / 0.0 -#endif - -typedef struct { - const unsigned char *json; - size_t position; -} error; -static error global_error = {NULL, 0}; - -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) { - return (const char *)(global_error.json + global_error.position); -} - -CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON *const item) { - if (!cJSON_IsString(item)) { - return NULL; - } - - return item->valuestring; -} - -CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON *const item) { - if (!cJSON_IsNumber(item)) { - return (double)NAN; - } - - return item->valuedouble; -} - -/* This is a safeguard to prevent copy-pasters from using incompatible C and - * header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || \ - (CJSON_VERSION_PATCH != 14) -#error cJSON.h and cJSON.c have different versions. Make sure that both have the same. -#endif - -CJSON_PUBLIC(const char *) cJSON_Version(void) { - static char version[15]; - sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, - CJSON_VERSION_PATCH); - - return version; -} - -/* Case insensitive string comparison, doesn't consider two NULL pointers equal - * though */ -static int case_insensitive_strcmp(const unsigned char *string1, - const unsigned char *string2) { - if ((string1 == NULL) || (string2 == NULL)) { - return 1; - } - - if (string1 == string2) { - return 0; - } - - for (; tolower(*string1) == tolower(*string2); - (void)string1++, string2++) { - if (*string1 == '\0') { - return 0; - } - } - - return tolower(*string1) - tolower(*string2); -} - -typedef struct internal_hooks { - void *(CJSON_CDECL *allocate)(size_t size); - void(CJSON_CDECL *deallocate)(void *pointer); - void *(CJSON_CDECL *reallocate)(void *pointer, size_t size); -} internal_hooks; - -#if defined(_MSC_VER) -/* work around MSVC error C2322: '...' address of dllimport '...' is not static - */ -static void *CJSON_CDECL internal_malloc(size_t size) { - return malloc(size); -} -static void CJSON_CDECL internal_free(void *pointer) { - free(pointer); -} -static void *CJSON_CDECL internal_realloc(void *pointer, size_t size) { - return realloc(pointer, size); -} -#else -#define internal_malloc malloc -#define internal_free free -#define internal_realloc realloc -#endif - -/* strlen of character literals resolved at compile time */ -#define static_strlen(string_literal) (sizeof(string_literal) - sizeof("")) - -static internal_hooks global_hooks = {internal_malloc, internal_free, - internal_realloc}; - -static unsigned char *cJSON_strdup(const unsigned char *string, - const internal_hooks *const hooks) { - size_t length = 0; - unsigned char *copy = NULL; - - if (string == NULL) { - return NULL; - } - - length = strlen((const char *)string) + sizeof(""); - copy = (unsigned char *)hooks->allocate(length); - if (copy == NULL) { - return NULL; - } - memcpy(copy, string, length); - - return copy; -} - -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks *hooks) { - if (hooks == NULL) { - /* Reset hooks */ - global_hooks.allocate = malloc; - global_hooks.deallocate = free; - global_hooks.reallocate = realloc; - return; - } - - global_hooks.allocate = malloc; - if (hooks->malloc_fn != NULL) { - global_hooks.allocate = hooks->malloc_fn; - } - - global_hooks.deallocate = free; - if (hooks->free_fn != NULL) { - global_hooks.deallocate = hooks->free_fn; - } - - /* use realloc only if both free and malloc are used */ - global_hooks.reallocate = NULL; - if ((global_hooks.allocate == malloc) && - (global_hooks.deallocate == free)) { - global_hooks.reallocate = realloc; - } -} - -/* Internal constructor. */ -static cJSON *cJSON_New_Item(const internal_hooks *const hooks) { - cJSON *node = (cJSON *)hooks->allocate(sizeof(cJSON)); - if (node) { - memset(node, '\0', sizeof(cJSON)); - } - - return node; -} - -/* Delete a cJSON structure. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) { - cJSON *next = NULL; - while (item != NULL) { - next = item->next; - if (!(item->type & cJSON_IsReference) && - (item->child != NULL)) { - cJSON_Delete(item->child); - } - if (!(item->type & cJSON_IsReference) && - (item->valuestring != NULL)) { - global_hooks.deallocate(item->valuestring); - } - if (!(item->type & cJSON_StringIsConst) && - (item->string != NULL)) { - global_hooks.deallocate(item->string); - } - global_hooks.deallocate(item); - item = next; - } -} - -/* get the decimal point character of the current locale */ -static unsigned char get_decimal_point(void) { -#ifdef ENABLE_LOCALES - struct lconv *lconv = localeconv(); - return (unsigned char)lconv->decimal_point[0]; -#else - return '.'; -#endif -} - -typedef struct { - const unsigned char *content; - size_t length; - size_t offset; - size_t depth; /* How deeply nested (in arrays/objects) is the input at - the current offset. */ - internal_hooks hooks; -} parse_buffer; - -/* check if the given size is left to read in a given parse buffer (starting - * with 1) */ -#define can_read(buffer, size) \ - ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) -/* check if the buffer can be accessed at the given index (starting with 0) */ -#define can_access_at_index(buffer, index) \ - ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) -#define cannot_access_at_index(buffer, index) \ - (!can_access_at_index(buffer, index)) -/* get a pointer to the buffer at the position */ -#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) - -/* Parse the input text to generate a number, and populate the result into item. - */ -static cJSON_bool parse_number(cJSON *const item, - parse_buffer *const input_buffer) { - double number = 0; - unsigned char *after_end = NULL; - unsigned char number_c_string[64]; - unsigned char decimal_point = get_decimal_point(); - size_t i = 0; - - if ((input_buffer == NULL) || (input_buffer->content == NULL)) { - return false; - } - - /* copy the number into a temporary buffer and replace '.' with the - * decimal point of the current locale (for strtod) - * This also takes care of '\0' not necessarily being available for - * marking the end of the input */ - for (i = 0; (i < (sizeof(number_c_string) - 1)) && - can_access_at_index(input_buffer, i); - i++) { - switch (buffer_at_offset(input_buffer)[i]) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '+': - case '-': - case 'e': - case 'E': - number_c_string[i] = buffer_at_offset(input_buffer)[i]; - break; - - case '.': - number_c_string[i] = decimal_point; - break; - - default: - goto loop_end; - } - } -loop_end: - number_c_string[i] = '\0'; - - number = strtod((const char *)number_c_string, (char **)&after_end); - if (number_c_string == after_end) { - return false; /* parse_error */ - } - - item->valuedouble = number; - - /* use saturation in case of overflow */ - if (number >= INT_MAX) { - item->valueint = INT_MAX; - } else if (number <= (double)INT_MIN) { - item->valueint = INT_MIN; - } else { - item->valueint = (int)number; - } - - item->type = cJSON_Number; - - input_buffer->offset += (size_t)(after_end - number_c_string); - return true; -} - -/* don't ask me, but the original cJSON_SetNumberValue returns an integer or - * double */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) { - if (number >= INT_MAX) { - object->valueint = INT_MAX; - } else if (number <= (double)INT_MIN) { - object->valueint = INT_MIN; - } else { - object->valueint = (int)number; - } - - return object->valuedouble = number; -} - -CJSON_PUBLIC(char *) -cJSON_SetValuestring(cJSON *object, const char *valuestring) { - char *copy = NULL; - /* if object's type is not cJSON_String or is cJSON_IsReference, it - * should not set valuestring */ - if (!(object->type & cJSON_String) || - (object->type & cJSON_IsReference)) { - return NULL; - } - if (strlen(valuestring) <= strlen(object->valuestring)) { - strcpy(object->valuestring, valuestring); - return object->valuestring; - } - copy = (char *)cJSON_strdup((const unsigned char *)valuestring, - &global_hooks); - if (copy == NULL) { - return NULL; - } - if (object->valuestring != NULL) { - cJSON_free(object->valuestring); - } - object->valuestring = copy; - - return copy; -} - -typedef struct { - unsigned char *buffer; - size_t length; - size_t offset; - size_t depth; /* current nesting depth (for formatted printing) */ - cJSON_bool noalloc; - cJSON_bool format; /* is this print a formatted print */ - internal_hooks hooks; -} printbuffer; - -/* realloc printbuffer if necessary to have at least "needed" bytes more */ -static unsigned char *ensure(printbuffer *const p, size_t needed) { - unsigned char *newbuffer = NULL; - size_t newsize = 0; - - if ((p == NULL) || (p->buffer == NULL)) { - return NULL; - } - - if ((p->length > 0) && (p->offset >= p->length)) { - /* make sure that offset is valid */ - return NULL; - } - - if (needed > INT_MAX) { - /* sizes bigger than INT_MAX are currently not supported */ - return NULL; - } - - needed += p->offset + 1; - if (needed <= p->length) { - return p->buffer + p->offset; - } - - if (p->noalloc) { - return NULL; - } - - /* calculate new buffer size */ - if (needed > (INT_MAX / 2)) { - /* overflow of int, use INT_MAX if possible */ - if (needed <= INT_MAX) { - newsize = INT_MAX; - } else { - return NULL; - } - } else { - newsize = needed * 2; - } - - if (p->hooks.reallocate != NULL) { - /* reallocate with realloc if available */ - newbuffer = - (unsigned char *)p->hooks.reallocate(p->buffer, newsize); - if (newbuffer == NULL) { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - } else { - /* otherwise reallocate manually */ - newbuffer = (unsigned char *)p->hooks.allocate(newsize); - if (!newbuffer) { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - if (newbuffer) { - memcpy(newbuffer, p->buffer, p->offset + 1); - } - p->hooks.deallocate(p->buffer); - } - p->length = newsize; - p->buffer = newbuffer; - - return newbuffer + p->offset; -} - -/* calculate the new length of the string in a printbuffer and update the offset - */ -static void update_offset(printbuffer *const buffer) { - const unsigned char *buffer_pointer = NULL; - if ((buffer == NULL) || (buffer->buffer == NULL)) { - return; - } - buffer_pointer = buffer->buffer + buffer->offset; - - buffer->offset += strlen((const char *)buffer_pointer); -} - -/* securely comparison of floating-point variables */ -static cJSON_bool compare_double(double a, double b) { - double maxVal = fabs(a) > fabs(b) ? fabs(a) : fabs(b); - return (fabs(a - b) <= maxVal * DBL_EPSILON); -} - -/* Render the number nicely from the given item into a string. */ -static cJSON_bool print_number(const cJSON *const item, - printbuffer *const output_buffer) { - unsigned char *output_pointer = NULL; - double d = item->valuedouble; - int length = 0; - size_t i = 0; - unsigned char number_buffer[26] = { - 0}; /* temporary buffer to print the number into */ - unsigned char decimal_point = get_decimal_point(); - double test = 0.0; - - if (output_buffer == NULL) { - return false; - } - - /* This checks for NaN and Infinity */ - if (isnan(d) || isinf(d)) { - length = sprintf((char *)number_buffer, "null"); - } else { - /* Try 15 decimal places of precision to avoid nonsignificant - * nonzero digits */ - length = sprintf((char *)number_buffer, "%1.15g", d); - - /* Check whether the original double can be recovered */ - if ((sscanf((char *)number_buffer, "%lg", &test) != 1) || - !compare_double((double)test, d)) { - /* If not, print with 17 decimal places of precision */ - length = sprintf((char *)number_buffer, "%1.17g", d); - } - } - - /* sprintf failed or buffer overrun occurred */ - if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) { - return false; - } - - /* reserve appropriate space in the output */ - output_pointer = ensure(output_buffer, (size_t)length + sizeof("")); - if (output_pointer == NULL) { - return false; - } - - /* copy the printed number to the output and replace locale - * dependent decimal point with '.' */ - for (i = 0; i < ((size_t)length); i++) { - if (number_buffer[i] == decimal_point) { - output_pointer[i] = '.'; - continue; - } - - output_pointer[i] = number_buffer[i]; - } - output_pointer[i] = '\0'; - - output_buffer->offset += (size_t)length; - - return true; -} - -/* parse 4 digit hexadecimal number */ -static unsigned parse_hex4(const unsigned char *const input) { - unsigned int h = 0; - size_t i = 0; - - for (i = 0; i < 4; i++) { - /* parse digit */ - if ((input[i] >= '0') && (input[i] <= '9')) { - h += (unsigned int)input[i] - '0'; - } else if ((input[i] >= 'A') && (input[i] <= 'F')) { - h += (unsigned int)10 + input[i] - 'A'; - } else if ((input[i] >= 'a') && (input[i] <= 'f')) { - h += (unsigned int)10 + input[i] - 'a'; - } else /* invalid */ - { - return 0; - } - - if (i < 3) { - /* shift left to make place for the next nibble */ - h = h << 4; - } - } - - return h; -} - -/* converts a UTF-16 literal to UTF-8 - * A literal can be one or two sequences of the form \uXXXX */ -static unsigned char -utf16_literal_to_utf8(const unsigned char *const input_pointer, - const unsigned char *const input_end, - unsigned char **output_pointer) { - long unsigned int codepoint = 0; - unsigned int first_code = 0; - const unsigned char *first_sequence = input_pointer; - unsigned char utf8_length = 0; - unsigned char utf8_position = 0; - unsigned char sequence_length = 0; - unsigned char first_byte_mark = 0; - - if ((input_end - first_sequence) < 6) { - /* input ends unexpectedly */ - goto fail; - } - - /* get the first utf16 sequence */ - first_code = parse_hex4(first_sequence + 2); - - /* check that the code is valid */ - if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) { - goto fail; - } - - /* UTF16 surrogate pair */ - if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) { - const unsigned char *second_sequence = first_sequence + 6; - unsigned int second_code = 0; - sequence_length = 12; /* \uXXXX\uXXXX */ - - if ((input_end - second_sequence) < 6) { - /* input ends unexpectedly */ - goto fail; - } - - if ((second_sequence[0] != '\\') || - (second_sequence[1] != 'u')) { - /* missing second half of the surrogate pair */ - goto fail; - } - - /* get the second utf16 sequence */ - second_code = parse_hex4(second_sequence + 2); - /* check that the code is valid */ - if ((second_code < 0xDC00) || (second_code > 0xDFFF)) { - /* invalid second half of the surrogate pair */ - goto fail; - } - - - /* calculate the unicode codepoint from the surrogate pair */ - codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | - (second_code & 0x3FF)); - } else { - sequence_length = 6; /* \uXXXX */ - codepoint = first_code; - } - - /* encode as UTF-8 - * takes at maximum 4 bytes to encode: - * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - if (codepoint < 0x80) { - /* normal ascii, encoding 0xxxxxxx */ - utf8_length = 1; - } else if (codepoint < 0x800) { - /* two bytes, encoding 110xxxxx 10xxxxxx */ - utf8_length = 2; - first_byte_mark = 0xC0; /* 11000000 */ - } else if (codepoint < 0x10000) { - /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ - utf8_length = 3; - first_byte_mark = 0xE0; /* 11100000 */ - } else if (codepoint <= 0x10FFFF) { - /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ - utf8_length = 4; - first_byte_mark = 0xF0; /* 11110000 */ - } else { - /* invalid unicode codepoint */ - goto fail; - } - - /* encode as utf8 */ - for (utf8_position = (unsigned char)(utf8_length - 1); - utf8_position > 0; utf8_position--) { - /* 10xxxxxx */ - (*output_pointer)[utf8_position] = - (unsigned char)((codepoint | 0x80) & 0xBF); - codepoint >>= 6; - } - /* encode first byte */ - if (utf8_length > 1) { - (*output_pointer)[0] = - (unsigned char)((codepoint | first_byte_mark) & 0xFF); - } else { - (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); - } - - *output_pointer += utf8_length; - - return sequence_length; - -fail: - return 0; -} - -/* Parse the input text into an unescaped cinput, and populate item. */ -static cJSON_bool parse_string(cJSON *const item, - parse_buffer *const input_buffer) { - const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; - const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; - unsigned char *output_pointer = NULL; - unsigned char *output = NULL; - - /* not a string */ - if (buffer_at_offset(input_buffer)[0] != '\"') { - goto fail; - } - - { - /* calculate approximate size of the output (overestimate) */ - size_t allocation_length = 0; - size_t skipped_bytes = 0; - while (((size_t)(input_end - input_buffer->content) < - input_buffer->length) && - (*input_end != '\"')) { - /* is escape sequence */ - if (input_end[0] == '\\') { - if ((size_t)(input_end + 1 - - input_buffer->content) >= - input_buffer->length) { - /* prevent buffer overflow when last - * input character is a backslash */ - goto fail; - } - skipped_bytes++; - input_end++; - } - input_end++; - } - if (((size_t)(input_end - input_buffer->content) >= - input_buffer->length) || - (*input_end != '\"')) { - goto fail; /* string ended unexpectedly */ - } - - /* This is at most how much we need for the output */ - allocation_length = - (size_t)(input_end - buffer_at_offset(input_buffer)) - - skipped_bytes; - output = (unsigned char *)input_buffer->hooks.allocate( - allocation_length + sizeof("")); - if (output == NULL) { - goto fail; /* allocation failure */ - } - } - - output_pointer = output; - /* loop through the string literal */ - while (input_pointer < input_end) { - if (*input_pointer != '\\') { - *output_pointer++ = *input_pointer++; - } - /* escape sequence */ - else { - unsigned char sequence_length = 2; - if ((input_end - input_pointer) < 1) { - goto fail; - } - - switch (input_pointer[1]) { - case 'b': - *output_pointer++ = '\b'; - break; - case 'f': - *output_pointer++ = '\f'; - break; - case 'n': - *output_pointer++ = '\n'; - break; - case 'r': - *output_pointer++ = '\r'; - break; - case 't': - *output_pointer++ = '\t'; - break; - case '\"': - case '\\': - case '/': - *output_pointer++ = input_pointer[1]; - break; - - /* UTF-16 literal */ - case 'u': - sequence_length = utf16_literal_to_utf8( - input_pointer, input_end, &output_pointer); - if (sequence_length == 0) { - /* failed to convert UTF16-literal to - * UTF-8 */ - goto fail; - } - break; - - default: - goto fail; - } - input_pointer += sequence_length; - } - } - - /* zero terminate the output */ - *output_pointer = '\0'; - - item->type = cJSON_String; - item->valuestring = (char *)output; - - input_buffer->offset = (size_t)(input_end - input_buffer->content); - input_buffer->offset++; - - return true; - -fail: - if (output != NULL) { - input_buffer->hooks.deallocate(output); - } - - if (input_pointer != NULL) { - input_buffer->offset = - (size_t)(input_pointer - input_buffer->content); - } - - return false; -} - -/* Render the cstring provided to an escaped version that can be printed. */ -static cJSON_bool print_string_ptr(const unsigned char *const input, - printbuffer *const output_buffer) { - const unsigned char *input_pointer = NULL; - unsigned char *output = NULL; - unsigned char *output_pointer = NULL; - size_t output_length = 0; - /* numbers of additional characters needed for escaping */ - size_t escape_characters = 0; - - if (output_buffer == NULL) { - return false; - } - - /* empty string */ - if (input == NULL) { - output = ensure(output_buffer, sizeof("\"\"")); - if (output == NULL) { - return false; - } - strcpy((char *)output, "\"\""); - - return true; - } - - /* set "flag" to 1 if something needs to be escaped */ - for (input_pointer = input; *input_pointer; input_pointer++) { - switch (*input_pointer) { - case '\"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - /* one character escape sequence */ - escape_characters++; - break; - default: - if (*input_pointer < 32) { - /* UTF-16 escape sequence uXXXX */ - escape_characters += 5; - } - break; - } - } - output_length = (size_t)(input_pointer - input) + escape_characters; - - output = ensure(output_buffer, output_length + sizeof("\"\"")); - if (output == NULL) { - return false; - } - - /* no characters have to be escaped */ - if (escape_characters == 0) { - output[0] = '\"'; - memcpy(output + 1, input, output_length); - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; - } - - output[0] = '\"'; - output_pointer = output + 1; - /* copy the string */ - for (input_pointer = input; *input_pointer != '\0'; - (void)input_pointer++, output_pointer++) { - if ((*input_pointer > 31) && (*input_pointer != '\"') && - (*input_pointer != '\\')) { - /* normal character, copy */ - *output_pointer = *input_pointer; - } else { - /* character needs to be escaped */ - *output_pointer++ = '\\'; - switch (*input_pointer) { - case '\\': - *output_pointer = '\\'; - break; - case '\"': - *output_pointer = '\"'; - break; - case '\b': - *output_pointer = 'b'; - break; - case '\f': - *output_pointer = 'f'; - break; - case '\n': - *output_pointer = 'n'; - break; - case '\r': - *output_pointer = 'r'; - break; - case '\t': - *output_pointer = 't'; - break; - default: - /* escape and print as unicode codepoint */ - sprintf((char *)output_pointer, "u%04x", - *input_pointer); - output_pointer += 4; - break; - } - } - } - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; -} - -/* Invoke print_string_ptr (which is useful) on an item. */ -static cJSON_bool print_string(const cJSON *const item, printbuffer *const p) { - return print_string_ptr((unsigned char *)item->valuestring, p); -} - -/* Predeclare these prototypes. */ -static cJSON_bool parse_value(cJSON *const item, - parse_buffer *const input_buffer); -static cJSON_bool print_value(const cJSON *const item, - printbuffer *const output_buffer); -static cJSON_bool parse_array(cJSON *const item, - parse_buffer *const input_buffer); -static cJSON_bool print_array(const cJSON *const item, - printbuffer *const output_buffer); -static cJSON_bool parse_object(cJSON *const item, - parse_buffer *const input_buffer); -static cJSON_bool print_object(const cJSON *const item, - printbuffer *const output_buffer); - -/* Utility to jump whitespace and cr/lf */ -static parse_buffer *buffer_skip_whitespace(parse_buffer *const buffer) { - if ((buffer == NULL) || (buffer->content == NULL)) { - return NULL; - } - - if (cannot_access_at_index(buffer, 0)) { - return buffer; - } - - while (can_access_at_index(buffer, 0) && - (buffer_at_offset(buffer)[0] <= 32)) { - buffer->offset++; - } - - if (buffer->offset == buffer->length) { - buffer->offset--; - } - - return buffer; -} - -/* skip the UTF-8 BOM (byte order mark) if it is at the beginning of a buffer */ -static parse_buffer *skip_utf8_bom(parse_buffer *const buffer) { - if ((buffer == NULL) || (buffer->content == NULL) || - (buffer->offset != 0)) { - return NULL; - } - - if (can_access_at_index(buffer, 4) && - (strncmp((const char *)buffer_at_offset(buffer), "\xEF\xBB\xBF", - 3) == 0)) { - buffer->offset += 3; - } - - return buffer; -} - -CJSON_PUBLIC(cJSON *) -cJSON_ParseWithOpts(const char *value, - const char **return_parse_end, - cJSON_bool require_null_terminated) { - size_t buffer_length; - - if (NULL == value) { - return NULL; - } - - /* Adding null character size due to require_null_terminated. */ - buffer_length = strlen(value) + sizeof(""); - - return cJSON_ParseWithLengthOpts(value, buffer_length, return_parse_end, - require_null_terminated); -} - -/* Parse an object - create a new root, and populate. */ -CJSON_PUBLIC(cJSON *) -cJSON_ParseWithLengthOpts(const char *value, - size_t buffer_length, - const char **return_parse_end, - cJSON_bool require_null_terminated) { - parse_buffer buffer = {0, 0, 0, 0, {0, 0, 0}}; - cJSON *item = NULL; - - /* reset error position */ - global_error.json = NULL; - global_error.position = 0; - - if (value == NULL || 0 == buffer_length) { - goto fail; - } - - buffer.content = (const unsigned char *)value; - buffer.length = buffer_length; - buffer.offset = 0; - buffer.hooks = global_hooks; - - item = cJSON_New_Item(&global_hooks); - if (item == NULL) /* memory fail */ - { - goto fail; - } - - if (!parse_value(item, - buffer_skip_whitespace(skip_utf8_bom(&buffer)))) { - /* parse failure. ep is set. */ - goto fail; - } - - /* if we require null-terminated JSON without appended garbage, skip and - * then check for a null terminator */ - if (require_null_terminated) { - buffer_skip_whitespace(&buffer); - if ((buffer.offset >= buffer.length) || - buffer_at_offset(&buffer)[0] != '\0') { - goto fail; - } - } - if (return_parse_end) { - *return_parse_end = (const char *)buffer_at_offset(&buffer); - } - - return item; - -fail: - if (item != NULL) { - cJSON_Delete(item); - } - - if (value != NULL) { - error local_error; - local_error.json = (const unsigned char *)value; - local_error.position = 0; - - if (buffer.offset < buffer.length) { - local_error.position = buffer.offset; - } else if (buffer.length > 0) { - local_error.position = buffer.length - 1; - } - - if (return_parse_end != NULL) { - *return_parse_end = (const char *)local_error.json + - local_error.position; - } - - global_error = local_error; - } - - return NULL; -} - -/* Default options for cJSON_Parse */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) { - return cJSON_ParseWithOpts(value, 0, 0); -} - -CJSON_PUBLIC(cJSON *) -cJSON_ParseWithLength(const char *value, size_t buffer_length) { - return cJSON_ParseWithLengthOpts(value, buffer_length, 0, 0); -} - -#define cjson_min(a, b) (((a) < (b)) ? (a) : (b)) - -static unsigned char *print(const cJSON *const item, - cJSON_bool format, - const internal_hooks *const hooks) { - static const size_t default_buffer_size = 256; - printbuffer buffer[1]; - unsigned char *printed = NULL; - - memset(buffer, 0, sizeof(buffer)); - - /* create buffer */ - buffer->buffer = (unsigned char *)hooks->allocate(default_buffer_size); - buffer->length = default_buffer_size; - buffer->format = format; - buffer->hooks = *hooks; - if (buffer->buffer == NULL) { - goto fail; - } - - /* print the value */ - if (!print_value(item, buffer)) { - goto fail; - } - update_offset(buffer); - - /* check if reallocate is available */ - if (hooks->reallocate != NULL) { - printed = (unsigned char *)hooks->reallocate( - buffer->buffer, buffer->offset + 1); - if (printed == NULL) { - goto fail; - } - buffer->buffer = NULL; - } else /* otherwise copy the JSON over to a new buffer */ - { - printed = (unsigned char *)hooks->allocate(buffer->offset + 1); - if (printed == NULL) { - goto fail; - } - memcpy(printed, buffer->buffer, - cjson_min(buffer->length, buffer->offset + 1)); - printed[buffer->offset] = '\0'; /* just to be sure */ - - /* free the buffer */ - hooks->deallocate(buffer->buffer); - } - - return printed; - -fail: - if (buffer->buffer != NULL) { - hooks->deallocate(buffer->buffer); - } - - if (printed != NULL) { - hooks->deallocate(printed); - } - - return NULL; -} - -/* Render a cJSON item/entity/structure to text. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) { - return (char *)print(item, true, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) { - return (char *)print(item, false, &global_hooks); -} - -CJSON_PUBLIC(char *) -cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) { - printbuffer p = {0, 0, 0, 0, 0, 0, {0, 0, 0}}; - - if (prebuffer < 0) { - return NULL; - } - - p.buffer = (unsigned char *)global_hooks.allocate((size_t)prebuffer); - if (!p.buffer) { - return NULL; - } - - p.length = (size_t)prebuffer; - p.offset = 0; - p.noalloc = false; - p.format = fmt; - p.hooks = global_hooks; - - if (!print_value(item, &p)) { - global_hooks.deallocate(p.buffer); - return NULL; - } - - return (char *)p.buffer; -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_PrintPreallocated(cJSON *item, - char *buffer, - const int length, - const cJSON_bool format) { - printbuffer p = {0, 0, 0, 0, 0, 0, {0, 0, 0}}; - - if ((length < 0) || (buffer == NULL)) { - return false; - } - - p.buffer = (unsigned char *)buffer; - p.length = (size_t)length; - p.offset = 0; - p.noalloc = true; - p.format = format; - p.hooks = global_hooks; - - return print_value(item, &p); -} - -/* Parser core - when encountering text, process appropriately. */ -static cJSON_bool parse_value(cJSON *const item, - parse_buffer *const input_buffer) { - if ((input_buffer == NULL) || (input_buffer->content == NULL)) { - return false; /* no input */ - } - - /* parse the different types of values */ - /* null */ - if (can_read(input_buffer, 4) && - (strncmp((const char *)buffer_at_offset(input_buffer), "null", 4) == - 0)) { - item->type = cJSON_NULL; - input_buffer->offset += 4; - return true; - } - /* false */ - if (can_read(input_buffer, 5) && - (strncmp((const char *)buffer_at_offset(input_buffer), "false", - 5) == 0)) { - item->type = cJSON_False; - input_buffer->offset += 5; - return true; - } - /* true */ - if (can_read(input_buffer, 4) && - (strncmp((const char *)buffer_at_offset(input_buffer), "true", 4) == - 0)) { - item->type = cJSON_True; - item->valueint = 1; - input_buffer->offset += 4; - return true; - } - /* string */ - if (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == '\"')) { - return parse_string(item, input_buffer); - } - /* number */ - if (can_access_at_index(input_buffer, 0) && - ((buffer_at_offset(input_buffer)[0] == '-') || - ((buffer_at_offset(input_buffer)[0] >= '0') && - (buffer_at_offset(input_buffer)[0] <= '9')))) { - return parse_number(item, input_buffer); - } - /* array */ - if (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == '[')) { - return parse_array(item, input_buffer); - } - /* object */ - if (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == '{')) { - return parse_object(item, input_buffer); - } - - return false; -} - -/* Render a value to text. */ -static cJSON_bool print_value(const cJSON *const item, - printbuffer *const output_buffer) { - unsigned char *output = NULL; - - if ((item == NULL) || (output_buffer == NULL)) { - return false; - } - - switch ((item->type) & 0xFF) { - case cJSON_NULL: - output = ensure(output_buffer, 5); - if (output == NULL) { - return false; - } - strcpy((char *)output, "null"); - return true; - - case cJSON_False: - output = ensure(output_buffer, 6); - if (output == NULL) { - return false; - } - strcpy((char *)output, "false"); - return true; - - case cJSON_True: - output = ensure(output_buffer, 5); - if (output == NULL) { - return false; - } - strcpy((char *)output, "true"); - return true; - - case cJSON_Number: - return print_number(item, output_buffer); - - case cJSON_Raw: { - size_t raw_length = 0; - if (item->valuestring == NULL) { - return false; - } - - raw_length = strlen(item->valuestring) + sizeof(""); - output = ensure(output_buffer, raw_length); - if (output == NULL) { - return false; - } - memcpy(output, item->valuestring, raw_length); - return true; - } - - case cJSON_String: - return print_string(item, output_buffer); - - case cJSON_Array: - return print_array(item, output_buffer); - - case cJSON_Object: - return print_object(item, output_buffer); - - default: - return false; - } -} - -/* Build an array from input text. */ -static cJSON_bool parse_array(cJSON *const item, - parse_buffer *const input_buffer) { - cJSON *head = NULL; /* head of the linked list */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (buffer_at_offset(input_buffer)[0] != '[') { - /* not an array */ - goto fail; - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == ']')) { - /* empty array */ - goto success; - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) { - /* start the linked list */ - current_item = head = new_item; - } else { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse next value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } while (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || - buffer_at_offset(input_buffer)[0] != ']') { - goto fail; /* expected end of array */ - } - -success: - input_buffer->depth--; - - if (head != NULL) { - head->prev = current_item; - } - - item->type = cJSON_Array; - item->child = head; - - input_buffer->offset++; - - return true; - -fail: - if (head != NULL) { - cJSON_Delete(head); - } - - return false; -} - -/* Render an array to text */ -static cJSON_bool print_array(const cJSON *const item, - printbuffer *const output_buffer) { - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_element = item->child; - - if (output_buffer == NULL) { - return false; - } - - /* Compose the output array. */ - /* opening square bracket */ - output_pointer = ensure(output_buffer, 1); - if (output_pointer == NULL) { - return false; - } - - *output_pointer = '['; - output_buffer->offset++; - output_buffer->depth++; - - while (current_element != NULL) { - if (!print_value(current_element, output_buffer)) { - return false; - } - update_offset(output_buffer); - if (current_element->next) { - length = (size_t)(output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) { - return false; - } - *output_pointer++ = ','; - if (output_buffer->format) { - *output_pointer++ = ' '; - } - *output_pointer = '\0'; - output_buffer->offset += length; - } - current_element = current_element->next; - } - - output_pointer = ensure(output_buffer, 2); - if (output_pointer == NULL) { - return false; - } - *output_pointer++ = ']'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Build an object from the text. */ -static cJSON_bool parse_object(cJSON *const item, - parse_buffer *const input_buffer) { - cJSON *head = NULL; /* linked list head */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (cannot_access_at_index(input_buffer, 0) || - (buffer_at_offset(input_buffer)[0] != '{')) { - goto fail; /* not an object */ - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == '}')) { - goto success; /* empty object */ - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) { - /* start the linked list */ - current_item = head = new_item; - } else { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse the name of the child */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_string(current_item, input_buffer)) { - goto fail; /* failed to parse name */ - } - buffer_skip_whitespace(input_buffer); - - /* swap valuestring and string, because we parsed the name */ - current_item->string = current_item->valuestring; - current_item->valuestring = NULL; - - if (cannot_access_at_index(input_buffer, 0) || - (buffer_at_offset(input_buffer)[0] != ':')) { - goto fail; /* invalid object */ - } - - /* parse the value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } while (can_access_at_index(input_buffer, 0) && - (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || - (buffer_at_offset(input_buffer)[0] != '}')) { - goto fail; /* expected end of object */ - } - -success: - input_buffer->depth--; - - if (head != NULL) { - head->prev = current_item; - } - - item->type = cJSON_Object; - item->child = head; - - input_buffer->offset++; - return true; - -fail: - if (head != NULL) { - cJSON_Delete(head); - } - - return false; -} - -/* Render an object to text. */ -static cJSON_bool print_object(const cJSON *const item, - printbuffer *const output_buffer) { - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_item = item->child; - - if (output_buffer == NULL) { - return false; - } - - /* Compose the output: */ - length = (size_t)(output_buffer->format ? 2 : 1); /* fmt: {\n */ - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) { - return false; - } - - *output_pointer++ = '{'; - output_buffer->depth++; - if (output_buffer->format) { - *output_pointer++ = '\n'; - } - output_buffer->offset += length; - - while (current_item) { - if (output_buffer->format) { - size_t i; - output_pointer = - ensure(output_buffer, output_buffer->depth); - if (output_pointer == NULL) { - return false; - } - for (i = 0; i < output_buffer->depth; i++) { - *output_pointer++ = '\t'; - } - output_buffer->offset += output_buffer->depth; - } - - /* print key */ - if (!print_string_ptr((unsigned char *)current_item->string, - output_buffer)) { - return false; - } - update_offset(output_buffer); - - length = (size_t)(output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length); - if (output_pointer == NULL) { - return false; - } - *output_pointer++ = ':'; - if (output_buffer->format) { - *output_pointer++ = '\t'; - } - output_buffer->offset += length; - - /* print value */ - if (!print_value(current_item, output_buffer)) { - return false; - } - update_offset(output_buffer); - - /* print comma if not last */ - length = ((size_t)(output_buffer->format ? 1 : 0) + - (size_t)(current_item->next ? 1 : 0)); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) { - return false; - } - if (current_item->next) { - *output_pointer++ = ','; - } - - if (output_buffer->format) { - *output_pointer++ = '\n'; - } - *output_pointer = '\0'; - output_buffer->offset += length; - - current_item = current_item->next; - } - - output_pointer = - ensure(output_buffer, - output_buffer->format ? (output_buffer->depth + 1) : 2); - if (output_pointer == NULL) { - return false; - } - if (output_buffer->format) { - size_t i; - for (i = 0; i < (output_buffer->depth - 1); i++) { - *output_pointer++ = '\t'; - } - } - *output_pointer++ = '}'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Get Array size/item / object item. */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) { - cJSON *child = NULL; - size_t size = 0; - - if (array == NULL) { - return 0; - } - - child = array->child; - - while (child != NULL) { - size++; - child = child->next; - } - - /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ - - return (int)size; -} - -static cJSON *get_array_item(const cJSON *array, size_t index) { - cJSON *current_child = NULL; - - if (array == NULL) { - return NULL; - } - - current_child = array->child; - while ((current_child != NULL) && (index > 0)) { - index--; - current_child = current_child->next; - } - - return current_child; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) { - if (index < 0) { - return NULL; - } - - return get_array_item(array, (size_t)index); -} - -static cJSON *get_object_item(const cJSON *const object, - const char *const name, - const cJSON_bool case_sensitive) { - cJSON *current_element = NULL; - - if ((object == NULL) || (name == NULL)) { - return NULL; - } - - current_element = object->child; - if (case_sensitive) { - while ((current_element != NULL) && - (current_element->string != NULL) && - (strcmp(name, current_element->string) != 0)) { - current_element = current_element->next; - } - } else { - while ((current_element != NULL) && - (case_insensitive_strcmp( - (const unsigned char *)name, - (const unsigned char *)(current_element->string)) != - 0)) { - current_element = current_element->next; - } - } - - if ((current_element == NULL) || (current_element->string == NULL)) { - return NULL; - } - - return current_element; -} - -CJSON_PUBLIC(cJSON *) -cJSON_GetObjectItem(const cJSON *const object, const char *const string) { - return get_object_item(object, string, false); -} - -CJSON_PUBLIC(cJSON *) -cJSON_GetObjectItemCaseSensitive(const cJSON *const object, - const char *const string) { - return get_object_item(object, string, true); -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_HasObjectItem(const cJSON *object, const char *string) { - return cJSON_GetObjectItem(object, string) ? 1 : 0; -} - -/* Utility for array list handling. */ -static void suffix_object(cJSON *prev, cJSON *item) { - prev->next = item; - item->prev = prev; -} - -/* Utility for handling references. */ -static cJSON *create_reference(const cJSON *item, - const internal_hooks *const hooks) { - cJSON *reference = NULL; - if (item == NULL) { - return NULL; - } - - reference = cJSON_New_Item(hooks); - if (reference == NULL) { - return NULL; - } - - memcpy(reference, item, sizeof(cJSON)); - reference->string = NULL; - reference->type |= cJSON_IsReference; - reference->next = reference->prev = NULL; - return reference; -} - -static cJSON_bool add_item_to_array(cJSON *array, cJSON *item) { - cJSON *child = NULL; - - if ((item == NULL) || (array == NULL) || (array == item)) { - return false; - } - - child = array->child; - /* - * To find the last item in array quickly, we use prev in array - */ - if (child == NULL) { - /* list is empty, start new one */ - array->child = item; - item->prev = item; - item->next = NULL; - } else { - /* append to the end */ - if (child->prev) { - suffix_object(child->prev, item); - array->child->prev = item; - } - } - - return true; -} - -/* Add item to array/object. */ -CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item) { - return add_item_to_array(array, item); -} - -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) -#pragma GCC diagnostic push -#endif -#ifdef __GNUC__ -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif -/* helper function to cast away const */ -static void *cast_away_const(const void *string) { - return (void *)string; -} -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) -#pragma GCC diagnostic pop -#endif - - -static cJSON_bool add_item_to_object(cJSON *const object, - const char *const string, - cJSON *const item, - const internal_hooks *const hooks, - const cJSON_bool constant_key) { - char *new_key = NULL; - int new_type = cJSON_Invalid; - - if ((object == NULL) || (string == NULL) || (item == NULL) || - (object == item)) { - return false; - } - - if (constant_key) { - new_key = (char *)cast_away_const(string); - new_type = item->type | cJSON_StringIsConst; - } else { - new_key = - (char *)cJSON_strdup((const unsigned char *)string, hooks); - if (new_key == NULL) { - return false; - } - - new_type = item->type & ~cJSON_StringIsConst; - } - - if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) { - hooks->deallocate(item->string); - } - - item->string = new_key; - item->type = new_type; - - return add_item_to_array(object, item); -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) { - return add_item_to_object(object, string, item, &global_hooks, false); -} - -/* Add an item to an object with constant string as key */ -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) { - return add_item_to_object(object, string, item, &global_hooks, true); -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) { - if (array == NULL) { - return false; - } - - return add_item_to_array(array, create_reference(item, &global_hooks)); -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) { - if ((object == NULL) || (string == NULL)) { - return false; - } - - return add_item_to_object(object, string, - create_reference(item, &global_hooks), - &global_hooks, false); -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddNullToObject(cJSON *const object, const char *const name) { - cJSON *null = cJSON_CreateNull(); - if (add_item_to_object(object, name, null, &global_hooks, false)) { - return null; - } - - cJSON_Delete(null); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddTrueToObject(cJSON *const object, const char *const name) { - cJSON *true_item = cJSON_CreateTrue(); - if (add_item_to_object(object, name, true_item, &global_hooks, false)) { - return true_item; - } - - cJSON_Delete(true_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddFalseToObject(cJSON *const object, const char *const name) { - cJSON *false_item = cJSON_CreateFalse(); - if (add_item_to_object(object, name, false_item, &global_hooks, - false)) { - return false_item; - } - - cJSON_Delete(false_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddBoolToObject(cJSON *const object, - const char *const name, - const cJSON_bool boolean) { - cJSON *bool_item = cJSON_CreateBool(boolean); - if (add_item_to_object(object, name, bool_item, &global_hooks, false)) { - return bool_item; - } - - cJSON_Delete(bool_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddNumberToObject(cJSON *const object, - const char *const name, - const double number) { - cJSON *number_item = cJSON_CreateNumber(number); - if (add_item_to_object(object, name, number_item, &global_hooks, - false)) { - return number_item; - } - - cJSON_Delete(number_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddStringToObject(cJSON *const object, - const char *const name, - const char *const string) { - cJSON *string_item = cJSON_CreateString(string); - if (add_item_to_object(object, name, string_item, &global_hooks, - false)) { - return string_item; - } - - cJSON_Delete(string_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddRawToObject(cJSON *const object, - const char *const name, - const char *const raw) { - cJSON *raw_item = cJSON_CreateRaw(raw); - if (add_item_to_object(object, name, raw_item, &global_hooks, false)) { - return raw_item; - } - - cJSON_Delete(raw_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddObjectToObject(cJSON *const object, const char *const name) { - cJSON *object_item = cJSON_CreateObject(); - if (add_item_to_object(object, name, object_item, &global_hooks, - false)) { - return object_item; - } - - cJSON_Delete(object_item); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_AddArrayToObject(cJSON *const object, const char *const name) { - cJSON *array = cJSON_CreateArray(); - if (add_item_to_object(object, name, array, &global_hooks, false)) { - return array; - } - - cJSON_Delete(array); - return NULL; -} - -CJSON_PUBLIC(cJSON *) -cJSON_DetachItemViaPointer(cJSON *parent, cJSON *const item) { - if ((parent == NULL) || (item == NULL)) { - return NULL; - } - - if (item != parent->child) { - /* not the first element */ - item->prev->next = item->next; - } - if (item->next != NULL) { - /* not the last element */ - item->next->prev = item->prev; - } - - if (item == parent->child) { - /* first element */ - parent->child = item->next; - } else if (item->next == NULL) { - /* last element */ - parent->child->prev = item->prev; - } - - /* make sure the detached item doesn't point anywhere anymore */ - item->prev = NULL; - item->next = NULL; - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) { - if (which < 0) { - return NULL; - } - - return cJSON_DetachItemViaPointer(array, - get_array_item(array, (size_t)which)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) { - cJSON_Delete(cJSON_DetachItemFromArray(array, which)); -} - -CJSON_PUBLIC(cJSON *) -cJSON_DetachItemFromObject(cJSON *object, const char *string) { - cJSON *to_detach = cJSON_GetObjectItem(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(cJSON *) -cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) { - cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(void) -cJSON_DeleteItemFromObject(cJSON *object, const char *string) { - cJSON_Delete(cJSON_DetachItemFromObject(object, string)); -} - -CJSON_PUBLIC(void) -cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) { - cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); -} - -/* Replace array/object items with new ones. */ -CJSON_PUBLIC(cJSON_bool) -cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) { - cJSON *after_inserted = NULL; - - if (which < 0) { - return false; - } - - after_inserted = get_array_item(array, (size_t)which); - if (after_inserted == NULL) { - return add_item_to_array(array, newitem); - } - - newitem->next = after_inserted; - newitem->prev = after_inserted->prev; - after_inserted->prev = newitem; - if (after_inserted == array->child) { - array->child = newitem; - } else { - newitem->prev->next = newitem; - } - return true; -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemViaPointer(cJSON *const parent, - cJSON *const item, - cJSON *replacement) { - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) { - return false; - } - - if (replacement == item) { - return true; - } - - replacement->next = item->next; - replacement->prev = item->prev; - - if (replacement->next != NULL) { - replacement->next->prev = replacement; - } - if (parent->child == item) { - if (parent->child->prev == parent->child) { - replacement->prev = replacement; - } - parent->child = replacement; - } else { /* - * To find the last item in array quickly, we use prev in - * array. We can't modify the last item's next pointer where - * this item was the parent's child - */ - if (replacement->prev != NULL) { - replacement->prev->next = replacement; - } - if (replacement->next == NULL) { - parent->child->prev = replacement; - } - } - - item->next = NULL; - item->prev = NULL; - cJSON_Delete(item); - - return true; -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) { - if (which < 0) { - return false; - } - - return cJSON_ReplaceItemViaPointer( - array, get_array_item(array, (size_t)which), newitem); -} - -static cJSON_bool replace_item_in_object(cJSON *object, - const char *string, - cJSON *replacement, - cJSON_bool case_sensitive) { - if ((replacement == NULL) || (string == NULL)) { - return false; - } - - /* replace the name in the replacement */ - if (!(replacement->type & cJSON_StringIsConst) && - (replacement->string != NULL)) { - cJSON_free(replacement->string); - } - replacement->string = - (char *)cJSON_strdup((const unsigned char *)string, &global_hooks); - replacement->type &= ~cJSON_StringIsConst; - - return cJSON_ReplaceItemViaPointer( - object, get_object_item(object, string, case_sensitive), - replacement); -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) { - return replace_item_in_object(object, string, newitem, false); -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, - const char *string, - cJSON *newitem) { - return replace_item_in_object(object, string, newitem, true); -} - -/* Create basic types: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_NULL; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_True; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = boolean ? cJSON_True : cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_Number; - item->valuedouble = num; - - /* use saturation in case of overflow */ - if (num >= INT_MAX) { - item->valueint = INT_MAX; - } else if (num <= (double)INT_MIN) { - item->valueint = INT_MIN; - } else { - item->valueint = (int)num; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_String; - item->valuestring = (char *)cJSON_strdup( - (const unsigned char *)string, &global_hooks); - if (!item->valuestring) { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item != NULL) { - item->type = cJSON_String | cJSON_IsReference; - item->valuestring = (char *)cast_away_const(string); - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item != NULL) { - item->type = cJSON_Object | cJSON_IsReference; - item->child = (cJSON *)cast_away_const(child); - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item != NULL) { - item->type = cJSON_Array | cJSON_IsReference; - item->child = (cJSON *)cast_away_const(child); - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_Raw; - item->valuestring = (char *)cJSON_strdup( - (const unsigned char *)raw, &global_hooks); - if (!item->valuestring) { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_Array; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) { - item->type = cJSON_Object; - } - - return item; -} - -/* Create Arrays: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) { - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) { - return NULL; - } - - a = cJSON_CreateArray(); - for (i = 0; a && (i < (size_t)count); i++) { - n = cJSON_CreateNumber(numbers[i]); - if (!n) { - cJSON_Delete(a); - return NULL; - } - if (!i) { - a->child = n; - } else { - suffix_object(p, n); - } - p = n; - } - a->child->prev = n; - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) { - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) { - return NULL; - } - - a = cJSON_CreateArray(); - - for (i = 0; a && (i < (size_t)count); i++) { - n = cJSON_CreateNumber((double)numbers[i]); - if (!n) { - cJSON_Delete(a); - return NULL; - } - if (!i) { - a->child = n; - } else { - suffix_object(p, n); - } - p = n; - } - a->child->prev = n; - - return a; -} - -CJSON_PUBLIC(cJSON *) -cJSON_CreateDoubleArray(const double *numbers, int count) { - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) { - return NULL; - } - - a = cJSON_CreateArray(); - - for (i = 0; a && (i < (size_t)count); i++) { - n = cJSON_CreateNumber(numbers[i]); - if (!n) { - cJSON_Delete(a); - return NULL; - } - if (!i) { - a->child = n; - } else { - suffix_object(p, n); - } - p = n; - } - a->child->prev = n; - - return a; -} - -CJSON_PUBLIC(cJSON *) -cJSON_CreateStringArray(const char *const *strings, int count) { - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (strings == NULL)) { - return NULL; - } - - a = cJSON_CreateArray(); - - for (i = 0; a && (i < (size_t)count); i++) { - n = cJSON_CreateString(strings[i]); - if (!n) { - cJSON_Delete(a); - return NULL; - } - if (!i) { - a->child = n; - } else { - suffix_object(p, n); - } - p = n; - } - a->child->prev = n; - - return a; -} - -/* Duplication */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) { - cJSON *newitem = NULL; - cJSON *child = NULL; - cJSON *next = NULL; - cJSON *newchild = NULL; - - /* Bail on bad ptr */ - if (!item) { - goto fail; - } - /* Create new item */ - newitem = cJSON_New_Item(&global_hooks); - if (!newitem) { - goto fail; - } - /* Copy over all vars */ - newitem->type = item->type & (~cJSON_IsReference); - newitem->valueint = item->valueint; - newitem->valuedouble = item->valuedouble; - if (item->valuestring) { - newitem->valuestring = (char *)cJSON_strdup( - (unsigned char *)item->valuestring, &global_hooks); - if (!newitem->valuestring) { - goto fail; - } - } - if (item->string) { - newitem->string = - (item->type & cJSON_StringIsConst) - ? item->string - : (char *)cJSON_strdup((unsigned char *)item->string, - &global_hooks); - if (!newitem->string) { - goto fail; - } - } - /* If non-recursive, then we're done! */ - if (!recurse) { - return newitem; - } - /* Walk the ->next chain for the child. */ - child = item->child; - while (child != NULL) { - newchild = cJSON_Duplicate( - child, true); /* Duplicate (with recurse) each item in the - ->next chain */ - if (!newchild) { - goto fail; - } - if (next != NULL) { - /* If newitem->child already set, then crosswire ->prev - * and ->next and move on */ - next->next = newchild; - newchild->prev = next; - next = newchild; - } else { - /* Set newitem->child and move to it */ - newitem->child = newchild; - next = newchild; - } - child = child->next; - } - if (newitem && newitem->child) { - newitem->child->prev = newchild; - } - - return newitem; - -fail: - if (newitem != NULL) { - cJSON_Delete(newitem); - } - - return NULL; -} - -static void skip_oneline_comment(char **input) { - *input += static_strlen("//"); - - for (; (*input)[0] != '\0'; ++(*input)) { - if ((*input)[0] == '\n') { - *input += static_strlen("\n"); - return; - } - } -} - -static void skip_multiline_comment(char **input) { - *input += static_strlen("/*"); - - for (; (*input)[0] != '\0'; ++(*input)) { - if (((*input)[0] == '*') && ((*input)[1] == '/')) { - *input += static_strlen("*/"); - return; - } - } -} - -static void minify_string(char **input, char **output) { - (*output)[0] = (*input)[0]; - *input += static_strlen("\""); - *output += static_strlen("\""); - - - for (; (*input)[0] != '\0'; (void)++(*input), ++(*output)) { - (*output)[0] = (*input)[0]; - - if ((*input)[0] == '\"') { - (*output)[0] = '\"'; - *input += static_strlen("\""); - *output += static_strlen("\""); - return; - } else if (((*input)[0] == '\\') && ((*input)[1] == '\"')) { - (*output)[1] = (*input)[1]; - *input += static_strlen("\""); - *output += static_strlen("\""); - } - } -} - -CJSON_PUBLIC(void) cJSON_Minify(char *json) { - char *into = json; - - if (json == NULL) { - return; - } - - while (json[0] != '\0') { - switch (json[0]) { - case ' ': - case '\t': - case '\r': - case '\n': - json++; - break; - - case '/': - if (json[1] == '/') { - skip_oneline_comment(&json); - } else if (json[1] == '*') { - skip_multiline_comment(&json); - } else { - json++; - } - break; - - case '\"': - minify_string(&json, (char **)&into); - break; - - default: - into[0] = json[0]; - json++; - into++; - } - } - - /* and null-terminate. */ - *into = '\0'; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_Invalid; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_False; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xff) == cJSON_True; -} - - -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & (cJSON_True | cJSON_False)) != 0; -} -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_NULL; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_Number; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_String; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_Array; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_Object; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON *const item) { - if (item == NULL) { - return false; - } - - return (item->type & 0xFF) == cJSON_Raw; -} - -CJSON_PUBLIC(cJSON_bool) -cJSON_Compare(const cJSON *const a, - const cJSON *const b, - const cJSON_bool case_sensitive) { - if ((a == NULL) || (b == NULL) || - ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) { - return false; - } - - /* check if type is valid */ - switch (a->type & 0xFF) { - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - case cJSON_Number: - case cJSON_String: - case cJSON_Raw: - case cJSON_Array: - case cJSON_Object: - break; - - default: - return false; - } - - /* identical objects are equal */ - if (a == b) { - return true; - } - - switch (a->type & 0xFF) { - /* in these cases and equal type is enough */ - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - return true; - - case cJSON_Number: - if (compare_double(a->valuedouble, b->valuedouble)) { - return true; - } - return false; - - case cJSON_String: - case cJSON_Raw: - if ((a->valuestring == NULL) || (b->valuestring == NULL)) { - return false; - } - if (strcmp(a->valuestring, b->valuestring) == 0) { - return true; - } - - return false; - - case cJSON_Array: { - cJSON *a_element = a->child; - cJSON *b_element = b->child; - - for (; (a_element != NULL) && (b_element != NULL);) { - if (!cJSON_Compare(a_element, b_element, - case_sensitive)) { - return false; - } - - a_element = a_element->next; - b_element = b_element->next; - } - - /* one of the arrays is longer than the other */ - if (a_element != b_element) { - return false; - } - - return true; - } - - case cJSON_Object: { - cJSON *a_element = NULL; - cJSON *b_element = NULL; - cJSON_ArrayForEach(a_element, a) { - /* TODO This has O(n^2) runtime, which is horrible! */ - b_element = get_object_item(b, a_element->string, - case_sensitive); - if (b_element == NULL) { - return false; - } - - if (!cJSON_Compare(a_element, b_element, - case_sensitive)) { - return false; - } - } - - /* doing this twice, once on a and b to prevent true comparison - * if a subset of b - * TODO: Do this the proper way, this is just a fix for now */ - cJSON_ArrayForEach(b_element, b) { - a_element = get_object_item(a, b_element->string, - case_sensitive); - if (a_element == NULL) { - return false; - } - - if (!cJSON_Compare(b_element, a_element, - case_sensitive)) { - return false; - } - } - - return true; - } - - default: - return false; - } -} - -CJSON_PUBLIC(void *) cJSON_malloc(size_t size) { - return global_hooks.allocate(size); -} - -CJSON_PUBLIC(void) cJSON_free(void *object) { - global_hooks.deallocate(object); -} diff --git a/lib/librdkafka-2.10.1/src/cJSON.h b/lib/librdkafka-2.10.1/src/cJSON.h deleted file mode 100644 index 1b5655c7b64..00000000000 --- a/lib/librdkafka-2.10.1/src/cJSON.h +++ /dev/null @@ -1,398 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#ifndef cJSON__h -#define cJSON__h - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(__WINDOWS__) && \ - (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) -#define __WINDOWS__ -#endif - -#ifdef __WINDOWS__ - -/* When compiling for windows, we specify a specific calling convention to avoid -issues where we are being called from a project with a different default calling -convention. For windows you have 3 define options: - -CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever -dllexport symbols CJSON_EXPORT_SYMBOLS - Define this on library build when you -want to dllexport symbols (default) CJSON_IMPORT_SYMBOLS - Define this if you -want to dllimport symbol - -For *nix builds that support visibility attribute, you can define similar -behavior by - -setting default visibility to hidden by adding --fvisibility=hidden (for gcc) -or --xldscope=hidden (for sun cc) -to CFLAGS - -then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way -CJSON_EXPORT_SYMBOLS does - -*/ - -#define CJSON_CDECL __cdecl -#define CJSON_STDCALL __stdcall - -/* export symbols by default, this is necessary for copy pasting the C and - * header file */ -#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && \ - !defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_EXPORT_SYMBOLS -#endif - -#if defined(CJSON_HIDE_SYMBOLS) -#define CJSON_PUBLIC(type) type CJSON_STDCALL -#elif defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllexport) type CJSON_STDCALL -#elif defined(CJSON_IMPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllimport) type CJSON_STDCALL -#endif -#else /* !__WINDOWS__ */ -#define CJSON_CDECL -#define CJSON_STDCALL - -#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined(__SUNPRO_C)) && \ - defined(CJSON_API_VISIBILITY) -#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type -#else -#define CJSON_PUBLIC(type) type -#endif -#endif - -/* project version */ -#define CJSON_VERSION_MAJOR 1 -#define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 14 - -#include - -/* cJSON Types: */ -#define cJSON_Invalid (0) -#define cJSON_False (1 << 0) -#define cJSON_True (1 << 1) -#define cJSON_NULL (1 << 2) -#define cJSON_Number (1 << 3) -#define cJSON_String (1 << 4) -#define cJSON_Array (1 << 5) -#define cJSON_Object (1 << 6) -#define cJSON_Raw (1 << 7) /* raw json */ - -#define cJSON_IsReference 256 -#define cJSON_StringIsConst 512 - -/* The cJSON structure: */ -typedef struct cJSON { - /* next/prev allow you to walk array/object chains. Alternatively, use - * GetArraySize/GetArrayItem/GetObjectItem */ - struct cJSON *next; - struct cJSON *prev; - /* An array or object item will have a child pointer pointing to a chain - * of the items in the array/object. */ - struct cJSON *child; - - /* The type of the item, as above. */ - int type; - - /* The item's string, if type==cJSON_String and type == cJSON_Raw */ - char *valuestring; - /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead - */ - int valueint; - /* The item's number, if type==cJSON_Number */ - double valuedouble; - - /* The item's name string, if this item is the child of, or is in the - * list of subitems of an object. */ - char *string; -} cJSON; - -typedef struct cJSON_Hooks { - /* malloc/free are CDECL on Windows regardless of the default calling - * convention of the compiler, so ensure the hooks allow passing those - * functions directly. */ - void *(CJSON_CDECL *malloc_fn)(size_t sz); - void(CJSON_CDECL *free_fn)(void *ptr); -} cJSON_Hooks; - -typedef int cJSON_bool; - -/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse - * them. This is to prevent stack overflows. */ -#ifndef CJSON_NESTING_LIMIT -#define CJSON_NESTING_LIMIT 1000 -#endif - -/* returns the version of cJSON as a string */ -CJSON_PUBLIC(const char *) cJSON_Version(void); - -/* Supply malloc, realloc and free functions to cJSON */ -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks *hooks); - -/* Memory Management: the caller is always responsible to free the results from - * all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib - * free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is - * cJSON_PrintPreallocated, where the caller has full responsibility of the - * buffer. */ -/* Supply a block of JSON, and this returns a cJSON object you can interrogate. - */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); -CJSON_PUBLIC(cJSON *) -cJSON_ParseWithLength(const char *value, size_t buffer_length); -/* ParseWithOpts allows you to require (and check) that the JSON is null - * terminated, and to retrieve the pointer to the final byte parsed. */ -/* If you supply a ptr in return_parse_end and parsing fails, then - * return_parse_end will contain a pointer to the error so will match - * cJSON_GetErrorPtr(). */ -CJSON_PUBLIC(cJSON *) -cJSON_ParseWithOpts(const char *value, - const char **return_parse_end, - cJSON_bool require_null_terminated); -CJSON_PUBLIC(cJSON *) -cJSON_ParseWithLengthOpts(const char *value, - size_t buffer_length, - const char **return_parse_end, - cJSON_bool require_null_terminated); - -/* Render a cJSON entity to text for transfer/storage. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); -/* Render a cJSON entity to text for transfer/storage without any formatting. */ -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); -/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess - * at the final size. guessing well reduces reallocation. fmt=0 gives - * unformatted, =1 gives formatted */ -CJSON_PUBLIC(char *) -cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); -/* Render a cJSON entity to text using a buffer already allocated in memory with - * given length. Returns 1 on success and 0 on failure. */ -/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will - * use, so to be safe allocate 5 bytes more than you actually need */ -CJSON_PUBLIC(cJSON_bool) -cJSON_PrintPreallocated(cJSON *item, - char *buffer, - const int length, - const cJSON_bool format); -/* Delete a cJSON entity and all subentities. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *item); - -/* Returns the number of items in an array (or object). */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); -/* Retrieve item number "index" from array "array". Returns NULL if - * unsuccessful. */ -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); -/* Get item "string" from object. Case insensitive. */ -CJSON_PUBLIC(cJSON *) -cJSON_GetObjectItem(const cJSON *const object, const char *const string); -CJSON_PUBLIC(cJSON *) -cJSON_GetObjectItemCaseSensitive(const cJSON *const object, - const char *const string); -CJSON_PUBLIC(cJSON_bool) -cJSON_HasObjectItem(const cJSON *object, const char *string); -/* For analysing failed parses. This returns a pointer to the parse error. - * You'll probably need to look a few chars back to make sense of it. Defined - * when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); - -/* Check item type and return its value */ -CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON *const item); -CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON *const item); - -/* These functions check the type of an item */ -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON *const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON *const item); - -/* These calls create a cJSON item of the appropriate type. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); -/* raw json */ -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); - -/* Create a string where valuestring references a string so - * it will not be freed by cJSON_Delete */ -CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string); -/* Create an object/array that only references it's elements so - * they will not be freed by cJSON_Delete */ -CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child); -CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child); - -/* These utilities create an Array of count items. - * The parameter count cannot be greater than the number of elements in the - * number array, otherwise array access will be out of bounds.*/ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); -CJSON_PUBLIC(cJSON *) -cJSON_CreateStringArray(const char *const *strings, int count); - -/* Append item to the specified array/object. */ -CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); -/* Use this when string is definitely const (i.e. a literal, or as good as), and - * will definitely survive the cJSON object. WARNING: When this function was - * used, make sure to always check that (item->type & cJSON_StringIsConst) is - * zero before writing to `item->string` */ -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); -/* Append reference to item to the specified array/object. Use this when you - * want to add an existing cJSON to a new cJSON, but don't want to corrupt your - * existing cJSON. */ -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(cJSON_bool) -cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); - -/* Remove/Detach items from Arrays/Objects. */ -CJSON_PUBLIC(cJSON *) -cJSON_DetachItemViaPointer(cJSON *parent, cJSON *const item); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(cJSON *) -cJSON_DetachItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(cJSON *) -cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); -CJSON_PUBLIC(void) -cJSON_DeleteItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(void) -cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); - -/* Update array items. */ -CJSON_PUBLIC(cJSON_bool) -cJSON_InsertItemInArray( - cJSON *array, - int which, - cJSON *newitem); /* Shifts pre-existing items to the right. */ -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemViaPointer(cJSON *const parent, - cJSON *const item, - cJSON *replacement); -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem); -CJSON_PUBLIC(cJSON_bool) -cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, - const char *string, - cJSON *newitem); - -/* Duplicate a cJSON item */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); -/* Duplicate will create a new, identical cJSON item to the one you pass, in new - * memory that will need to be released. With recurse!=0, it will duplicate any - * children connected to the item. - * The item->next and ->prev pointers are always zero on return from Duplicate. - */ -/* Recursively compare two cJSON items for equality. If either a or b is NULL or - * invalid, they will be considered unequal. - * case_sensitive determines if object keys are treated case sensitive (1) or - * case insensitive (0) */ -CJSON_PUBLIC(cJSON_bool) -cJSON_Compare(const cJSON *const a, - const cJSON *const b, - const cJSON_bool case_sensitive); - -/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') from - * strings. The input pointer json cannot point to a read-only address area, - * such as a string constant, - * but should point to a readable and writable adress area. */ -CJSON_PUBLIC(void) cJSON_Minify(char *json); - -/* Helper functions for creating and adding items to an object at the same time. - * They return the added item or NULL on failure. */ -CJSON_PUBLIC(cJSON *) -cJSON_AddNullToObject(cJSON *const object, const char *const name); -CJSON_PUBLIC(cJSON *) -cJSON_AddTrueToObject(cJSON *const object, const char *const name); -CJSON_PUBLIC(cJSON *) -cJSON_AddFalseToObject(cJSON *const object, const char *const name); -CJSON_PUBLIC(cJSON *) -cJSON_AddBoolToObject(cJSON *const object, - const char *const name, - const cJSON_bool boolean); -CJSON_PUBLIC(cJSON *) -cJSON_AddNumberToObject(cJSON *const object, - const char *const name, - const double number); -CJSON_PUBLIC(cJSON *) -cJSON_AddStringToObject(cJSON *const object, - const char *const name, - const char *const string); -CJSON_PUBLIC(cJSON *) -cJSON_AddRawToObject(cJSON *const object, - const char *const name, - const char *const raw); -CJSON_PUBLIC(cJSON *) -cJSON_AddObjectToObject(cJSON *const object, const char *const name); -CJSON_PUBLIC(cJSON *) -cJSON_AddArrayToObject(cJSON *const object, const char *const name); - -/* When assigning an integer value, it needs to be propagated to valuedouble - * too. */ -#define cJSON_SetIntValue(object, number) \ - ((object) ? (object)->valueint = (object)->valuedouble = (number) \ - : (number)) -/* helper for the cJSON_SetNumberValue macro */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); -#define cJSON_SetNumberValue(object, number) \ - ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) \ - : (number)) -/* Change the valuestring of a cJSON_String object, only takes effect when type - * of object is cJSON_String */ -CJSON_PUBLIC(char *) -cJSON_SetValuestring(cJSON *object, const char *valuestring); - -/* Macro for iterating over an array or object */ -#define cJSON_ArrayForEach(element, array) \ - for (element = (array != NULL) ? (array)->child : NULL; \ - element != NULL; element = element->next) - -/* malloc/free objects using the malloc/free functions that have been set with - * cJSON_InitHooks */ -CJSON_PUBLIC(void *) cJSON_malloc(size_t size); -CJSON_PUBLIC(void) cJSON_free(void *object); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/lib/librdkafka-2.10.1/src/rdhttp.c b/lib/librdkafka-2.10.1/src/rdhttp.c deleted file mode 100644 index e5a9ad276f8..00000000000 --- a/lib/librdkafka-2.10.1/src/rdhttp.c +++ /dev/null @@ -1,519 +0,0 @@ -/* - * librdkafka - The Apache Kafka C/C++ library - * - * Copyright (c) 2021-2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -/** - * @name HTTP client - * - */ - -#include "rdkafka_int.h" -#include "rdunittest.h" - -#include - -#include -#include "rdhttp.h" - -/** Maximum response size, increase as necessary. */ -#define RD_HTTP_RESPONSE_SIZE_MAX 1024 * 1024 * 500 /* 500kb */ - - -void rd_http_error_destroy(rd_http_error_t *herr) { - rd_free(herr); -} - -static rd_http_error_t *rd_http_error_new(int code, const char *fmt, ...) - RD_FORMAT(printf, 2, 3); -static rd_http_error_t *rd_http_error_new(int code, const char *fmt, ...) { - size_t len = 0; - rd_http_error_t *herr; - va_list ap; - - va_start(ap, fmt); - - if (fmt && *fmt) { - va_list ap2; - va_copy(ap2, ap); - len = rd_vsnprintf(NULL, 0, fmt, ap2); - va_end(ap2); - } - - /* Use single allocation for both herr and the error string */ - herr = rd_malloc(sizeof(*herr) + len + 1); - herr->code = code; - herr->errstr = herr->data; - - if (len > 0) - rd_vsnprintf(herr->errstr, len + 1, fmt, ap); - else - herr->errstr[0] = '\0'; - - va_end(ap); - - return herr; -} - -/** - * @brief Same as rd_http_error_new() but reads the error string from the - * provided buffer. - */ -static rd_http_error_t *rd_http_error_new_from_buf(int code, - const rd_buf_t *rbuf) { - rd_http_error_t *herr; - rd_slice_t slice; - size_t len = rd_buf_len(rbuf); - - if (len == 0) - return rd_http_error_new( - code, "Server did not provide an error string"); - - - /* Use single allocation for both herr and the error string */ - herr = rd_malloc(sizeof(*herr) + len + 1); - herr->code = code; - herr->errstr = herr->data; - rd_slice_init_full(&slice, rbuf); - rd_slice_read(&slice, herr->errstr, len); - herr->errstr[len] = '\0'; - - return herr; -} - -void rd_http_req_destroy(rd_http_req_t *hreq) { - RD_IF_FREE(hreq->hreq_curl, curl_easy_cleanup); - RD_IF_FREE(hreq->hreq_buf, rd_buf_destroy_free); -} - - -/** - * @brief Curl writefunction. Writes the bytes passed from curl - * to the hreq's buffer. - */ -static size_t -rd_http_req_write_cb(char *ptr, size_t size, size_t nmemb, void *userdata) { - rd_http_req_t *hreq = (rd_http_req_t *)userdata; - - if (unlikely(rd_buf_len(hreq->hreq_buf) + nmemb > - RD_HTTP_RESPONSE_SIZE_MAX)) - return 0; /* FIXME: Set some overflow flag or rely on curl? */ - - rd_buf_write(hreq->hreq_buf, ptr, nmemb); - - return nmemb; -} - -rd_http_error_t *rd_http_req_init(rd_http_req_t *hreq, const char *url) { - - memset(hreq, 0, sizeof(*hreq)); - - hreq->hreq_curl = curl_easy_init(); - if (!hreq->hreq_curl) - return rd_http_error_new(-1, "Failed to create curl handle"); - - hreq->hreq_buf = rd_buf_new(1, 1024); - - curl_easy_setopt(hreq->hreq_curl, CURLOPT_URL, url); -#if CURL_AT_LEAST_VERSION(7, 85, 0) - curl_easy_setopt(hreq->hreq_curl, CURLOPT_PROTOCOLS_STR, "http,https"); -#else - /* As of 06/10/2025 Debian 10 and CentOS Stream 9 ship with - * older CURL versions, remove this condition once they're not supported - * anymore. */ - curl_easy_setopt(hreq->hreq_curl, CURLOPT_PROTOCOLS, - CURLPROTO_HTTP | CURLPROTO_HTTPS); -#endif - curl_easy_setopt(hreq->hreq_curl, CURLOPT_MAXREDIRS, 16); - curl_easy_setopt(hreq->hreq_curl, CURLOPT_TIMEOUT, 30); - curl_easy_setopt(hreq->hreq_curl, CURLOPT_ERRORBUFFER, - hreq->hreq_curl_errstr); - curl_easy_setopt(hreq->hreq_curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(hreq->hreq_curl, CURLOPT_WRITEFUNCTION, - rd_http_req_write_cb); - curl_easy_setopt(hreq->hreq_curl, CURLOPT_WRITEDATA, (void *)hreq); - - return NULL; -} - -/** - * @brief Synchronously (blockingly) perform the HTTP operation. - */ -rd_http_error_t *rd_http_req_perform_sync(rd_http_req_t *hreq) { - CURLcode res; - long code = 0; - - res = curl_easy_perform(hreq->hreq_curl); - if (unlikely(res != CURLE_OK)) - return rd_http_error_new(-1, "%s", hreq->hreq_curl_errstr); - - curl_easy_getinfo(hreq->hreq_curl, CURLINFO_RESPONSE_CODE, &code); - hreq->hreq_code = (int)code; - if (hreq->hreq_code >= 400) - return rd_http_error_new_from_buf(hreq->hreq_code, - hreq->hreq_buf); - - return NULL; -} - - -int rd_http_req_get_code(const rd_http_req_t *hreq) { - return hreq->hreq_code; -} - -const char *rd_http_req_get_content_type(rd_http_req_t *hreq) { - const char *content_type = NULL; - - if (curl_easy_getinfo(hreq->hreq_curl, CURLINFO_CONTENT_TYPE, - &content_type)) - return NULL; - - return content_type; -} - - -/** - * @brief Perform a blocking HTTP(S) request to \p url. - * - * Returns the response (even if there's a HTTP error code returned) - * in \p *rbufp. - * - * Returns NULL on success (HTTP response code < 400), or an error - * object on transport or HTTP error - this error object must be destroyed - * by calling rd_http_error_destroy(). In case of HTTP error the \p *rbufp - * may be filled with the error response. - */ -rd_http_error_t *rd_http_get(const char *url, rd_buf_t **rbufp) { - rd_http_req_t hreq; - rd_http_error_t *herr; - - *rbufp = NULL; - - herr = rd_http_req_init(&hreq, url); - if (unlikely(herr != NULL)) - return herr; - - herr = rd_http_req_perform_sync(&hreq); - if (herr) { - rd_http_req_destroy(&hreq); - return herr; - } - - *rbufp = hreq.hreq_buf; - hreq.hreq_buf = NULL; - - return NULL; -} - - -/** - * @brief Extract the JSON object from \p hreq and return it in \p *jsonp. - * - * @returns Returns NULL on success, or an JSON parsing error - this - * error object must be destroyed by calling rd_http_error_destroy(). - */ -rd_http_error_t *rd_http_parse_json(rd_http_req_t *hreq, cJSON **jsonp) { - size_t len; - char *raw_json; - const char *end = NULL; - rd_slice_t slice; - rd_http_error_t *herr = NULL; - - /* cJSON requires the entire input to parse in contiguous memory. */ - rd_slice_init_full(&slice, hreq->hreq_buf); - len = rd_buf_len(hreq->hreq_buf); - - raw_json = rd_malloc(len + 1); - rd_slice_read(&slice, raw_json, len); - raw_json[len] = '\0'; - - /* Parse JSON */ - *jsonp = cJSON_ParseWithOpts(raw_json, &end, 0); - - if (!*jsonp) - herr = rd_http_error_new(hreq->hreq_code, - "Failed to parse JSON response " - "at %" PRIusz "/%" PRIusz, - (size_t)(end - raw_json), len); - rd_free(raw_json); - return herr; -} - - -/** - * @brief Check if the error returned from HTTP(S) is temporary or not. - * - * @returns If the \p error_code is temporary, return rd_true, - * otherwise return rd_false. - * - * @locality Any thread. - */ -static rd_bool_t rd_http_is_failure_temporary(int error_code) { - switch (error_code) { - case 408: /**< Request timeout */ - case 425: /**< Too early */ - case 429: /**< Too many requests */ - case 500: /**< Internal server error */ - case 502: /**< Bad gateway */ - case 503: /**< Service unavailable */ - case 504: /**< Gateway timeout */ - return rd_true; - - default: - return rd_false; - } -} - - -/** - * @brief Perform a blocking HTTP(S) request to \p url with - * HTTP(S) headers and data with \p timeout_s. - * If the HTTP(S) request fails, will retry another \p retries times - * with multiplying backoff \p retry_ms. - * - * @returns The result will be returned in \p *jsonp. - * Returns NULL on success (HTTP response code < 400), or an error - * object on transport, HTTP error or a JSON parsing error - this - * error object must be destroyed by calling rd_http_error_destroy(). - * - * @locality Any thread. - */ -rd_http_error_t *rd_http_post_expect_json(rd_kafka_t *rk, - const char *url, - const struct curl_slist *headers, - const char *post_fields, - size_t post_fields_size, - int timeout_s, - int retries, - int retry_ms, - cJSON **jsonp) { - rd_http_error_t *herr; - rd_http_req_t hreq; - int i; - size_t len; - const char *content_type; - - herr = rd_http_req_init(&hreq, url); - if (unlikely(herr != NULL)) - return herr; - - curl_easy_setopt(hreq.hreq_curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(hreq.hreq_curl, CURLOPT_TIMEOUT, timeout_s); - - curl_easy_setopt(hreq.hreq_curl, CURLOPT_POSTFIELDSIZE, - post_fields_size); - curl_easy_setopt(hreq.hreq_curl, CURLOPT_POSTFIELDS, post_fields); - - for (i = 0; i <= retries; i++) { - if (rd_kafka_terminating(rk)) { - rd_http_req_destroy(&hreq); - return rd_http_error_new(-1, "Terminating"); - } - - herr = rd_http_req_perform_sync(&hreq); - len = rd_buf_len(hreq.hreq_buf); - - if (!herr) { - if (len > 0) - break; /* Success */ - /* Empty response */ - rd_http_req_destroy(&hreq); - return NULL; - } - /* Retry if HTTP(S) request returns temporary error and there - * are remaining retries, else fail. */ - if (i == retries || !rd_http_is_failure_temporary(herr->code)) { - rd_http_req_destroy(&hreq); - return herr; - } - - /* Retry */ - rd_http_error_destroy(herr); - rd_usleep(retry_ms * 1000 * (i + 1), &rk->rk_terminate); - } - - content_type = rd_http_req_get_content_type(&hreq); - - if (!content_type || rd_strncasecmp(content_type, "application/json", - strlen("application/json"))) { - if (!herr) - herr = rd_http_error_new( - hreq.hreq_code, "Response is not JSON encoded: %s", - content_type ? content_type : "(n/a)"); - rd_http_req_destroy(&hreq); - return herr; - } - - herr = rd_http_parse_json(&hreq, jsonp); - - rd_http_req_destroy(&hreq); - - return herr; -} - - -/** - * @brief Same as rd_http_get() but requires a JSON response. - * The response is parsed and a JSON object is returned in \p *jsonp. - * - * Same error semantics as rd_http_get(). - */ -rd_http_error_t *rd_http_get_json(const char *url, cJSON **jsonp) { - rd_http_req_t hreq; - rd_http_error_t *herr; - rd_slice_t slice; - size_t len; - const char *content_type; - char *raw_json; - const char *end; - - *jsonp = NULL; - - herr = rd_http_req_init(&hreq, url); - if (unlikely(herr != NULL)) - return herr; - - // FIXME: send Accept: json.. header? - - herr = rd_http_req_perform_sync(&hreq); - len = rd_buf_len(hreq.hreq_buf); - if (herr && len == 0) { - rd_http_req_destroy(&hreq); - return herr; - } - - if (len == 0) { - /* Empty response: create empty JSON object */ - *jsonp = cJSON_CreateObject(); - rd_http_req_destroy(&hreq); - return NULL; - } - - content_type = rd_http_req_get_content_type(&hreq); - - if (!content_type || rd_strncasecmp(content_type, "application/json", - strlen("application/json"))) { - if (!herr) - herr = rd_http_error_new( - hreq.hreq_code, "Response is not JSON encoded: %s", - content_type ? content_type : "(n/a)"); - rd_http_req_destroy(&hreq); - return herr; - } - - /* cJSON requires the entire input to parse in contiguous memory. */ - rd_slice_init_full(&slice, hreq.hreq_buf); - raw_json = rd_malloc(len + 1); - rd_slice_read(&slice, raw_json, len); - raw_json[len] = '\0'; - - /* Parse JSON */ - end = NULL; - *jsonp = cJSON_ParseWithOpts(raw_json, &end, 0); - if (!*jsonp && !herr) - herr = rd_http_error_new(hreq.hreq_code, - "Failed to parse JSON response " - "at %" PRIusz "/%" PRIusz, - (size_t)(end - raw_json), len); - - rd_free(raw_json); - rd_http_req_destroy(&hreq); - - return herr; -} - - -void rd_http_global_init(void) { - curl_global_init(CURL_GLOBAL_DEFAULT); -} - - -/** - * @brief Unittest. Requires a (local) webserver to be set with env var - * RD_UT_HTTP_URL=http://localhost:1234/some-path - * - * This server must return a JSON object or array containing at least one - * object on the main URL with a 2xx response code, - * and 4xx response on $RD_UT_HTTP_URL/error (with whatever type of body). - */ - -int unittest_http(void) { - const char *base_url = rd_getenv("RD_UT_HTTP_URL", NULL); - char *error_url; - size_t error_url_size; - cJSON *json, *jval; - rd_http_error_t *herr; - rd_bool_t empty; - - if (!base_url || !*base_url) - RD_UT_SKIP("RD_UT_HTTP_URL environment variable not set"); - - RD_UT_BEGIN(); - - error_url_size = strlen(base_url) + strlen("/error") + 1; - error_url = rd_alloca(error_url_size); - rd_snprintf(error_url, error_url_size, "%s/error", base_url); - - /* Try the base url first, parse its JSON and extract a key-value. */ - json = NULL; - herr = rd_http_get_json(base_url, &json); - RD_UT_ASSERT(!herr, "Expected get_json(%s) to succeed, got: %s", - base_url, herr->errstr); - - empty = rd_true; - cJSON_ArrayForEach(jval, json) { - empty = rd_false; - break; - } - RD_UT_ASSERT(!empty, "Expected non-empty JSON response from %s", - base_url); - RD_UT_SAY( - "URL %s returned no error and a non-empty " - "JSON object/array as expected", - base_url); - cJSON_Delete(json); - - - /* Try the error URL, verify error code. */ - json = NULL; - herr = rd_http_get_json(error_url, &json); - RD_UT_ASSERT(herr != NULL, "Expected get_json(%s) to fail", error_url); - RD_UT_ASSERT(herr->code >= 400, - "Expected get_json(%s) error code >= " - "400, got %d", - error_url, herr->code); - RD_UT_SAY( - "Error URL %s returned code %d, errstr \"%s\" " - "and %s JSON object as expected", - error_url, herr->code, herr->errstr, json ? "a" : "no"); - /* Check if there's a JSON document returned */ - if (json) - cJSON_Delete(json); - rd_http_error_destroy(herr); - - RD_UT_PASS(); -} diff --git a/lib/librdkafka-2.10.1/src/rdkafka_fetcher.c b/lib/librdkafka-2.10.1/src/rdkafka_fetcher.c deleted file mode 100644 index e275ee5a88b..00000000000 --- a/lib/librdkafka-2.10.1/src/rdkafka_fetcher.c +++ /dev/null @@ -1,1422 +0,0 @@ -/* - * librdkafka - The Apache Kafka C/C++ library - * - * Copyright (c) 2022, Magnus Edenhill - * 2023, Confluent Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -/** - * @name Fetcher - * - */ - -#include "rdkafka_int.h" -#include "rdkafka_offset.h" -#include "rdkafka_msgset.h" -#include "rdkafka_fetcher.h" -#include "rdkafka_request.h" - - -/** - * Backoff the next Fetch request (due to error). - */ -static void rd_kafka_broker_fetch_backoff(rd_kafka_broker_t *rkb, - rd_kafka_resp_err_t err) { - int backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms; - rkb->rkb_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000); - rd_rkb_dbg(rkb, FETCH, "BACKOFF", "Fetch backoff for %dms: %s", - backoff_ms, rd_kafka_err2str(err)); -} - -/** - * @brief Backoff the next Fetch for specific partition - * - * @returns the absolute backoff time (the current time for no backoff). - */ -static rd_ts_t rd_kafka_toppar_fetch_backoff(rd_kafka_broker_t *rkb, - rd_kafka_toppar_t *rktp, - rd_kafka_resp_err_t err) { - int backoff_ms; - - /* Don't back off on reaching end of partition */ - if (err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { - rktp->rktp_ts_fetch_backoff = 0; - return rd_clock(); /* Immediate: No practical backoff */ - } - - if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) - backoff_ms = rkb->rkb_rk->rk_conf.fetch_queue_backoff_ms; - else - backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms; - - if (unlikely(!backoff_ms)) { - rktp->rktp_ts_fetch_backoff = 0; - return rd_clock(); /* Immediate: No practical backoff */ - } - - /* Certain errors that may require manual intervention should have - * a longer backoff time. */ - if (err == RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED) - backoff_ms = RD_MAX(1000, backoff_ms * 10); - - rktp->rktp_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000); - - rd_rkb_dbg(rkb, FETCH, "BACKOFF", - "%s [%" PRId32 "]: Fetch backoff for %dms%s%s", - rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, - backoff_ms, err ? ": " : "", - err ? rd_kafka_err2str(err) : ""); - - return rktp->rktp_ts_fetch_backoff; -} - -/** - * @brief Handle preferred replica in fetch response. - * - * @locks rd_kafka_toppar_lock(rktp) and - * rd_kafka_rdlock(rk) must NOT be held. - * - * @locality broker thread - */ -static void rd_kafka_fetch_preferred_replica_handle(rd_kafka_toppar_t *rktp, - rd_kafka_buf_t *rkbuf, - rd_kafka_broker_t *rkb, - int32_t preferred_id) { - const rd_ts_t one_minute = 60 * 1000 * 1000; - const rd_ts_t five_seconds = 5 * 1000 * 1000; - rd_kafka_broker_t *preferred_rkb; - rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk; - rd_ts_t new_intvl = - rd_interval_immediate(&rktp->rktp_new_lease_intvl, one_minute, 0); - - if (new_intvl < 0) { - /* In lieu of KIP-320, the toppar is delegated back to - * the leader in the event of an offset out-of-range - * error (KIP-392 error case #4) because this scenario - * implies the preferred replica is out-of-sync. - * - * If program execution reaches here, the leader has - * relatively quickly instructed the client back to - * a preferred replica, quite possibly the same one - * as before (possibly resulting from stale metadata), - * so we back off the toppar to slow down potential - * back-and-forth. - */ - - if (rd_interval_immediate(&rktp->rktp_new_lease_log_intvl, - one_minute, 0) > 0) - rd_rkb_log(rkb, LOG_NOTICE, "FETCH", - "%.*s [%" PRId32 - "]: preferred replica " - "(%" PRId32 - ") lease changing too quickly " - "(%" PRId64 - "s < 60s): possibly due to " - "unavailable replica or stale cluster " - "state: backing off next fetch", - RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), - rktp->rktp_partition, preferred_id, - (one_minute - -new_intvl) / (1000 * 1000)); - - rd_kafka_toppar_fetch_backoff(rkb, rktp, - RD_KAFKA_RESP_ERR_NO_ERROR); - } - - rd_kafka_rdlock(rk); - preferred_rkb = rd_kafka_broker_find_by_nodeid(rk, preferred_id); - rd_kafka_rdunlock(rk); - - if (preferred_rkb) { - rd_interval_reset_to_now(&rktp->rktp_lease_intvl, 0); - rd_kafka_toppar_lock(rktp); - rd_kafka_toppar_broker_update(rktp, preferred_id, preferred_rkb, - "preferred replica updated"); - rd_kafka_toppar_unlock(rktp); - rd_kafka_broker_destroy(preferred_rkb); - return; - } - - if (rd_interval_immediate(&rktp->rktp_metadata_intvl, five_seconds, 0) > - 0) { - rd_rkb_log(rkb, LOG_NOTICE, "FETCH", - "%.*s [%" PRId32 "]: preferred replica (%" PRId32 - ") " - "is unknown: refreshing metadata", - RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), - rktp->rktp_partition, preferred_id); - - rd_kafka_metadata_refresh_brokers( - rktp->rktp_rkt->rkt_rk, NULL, - "preferred replica unavailable"); - } - - rd_kafka_toppar_fetch_backoff(rkb, rktp, - RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE); -} - - -/** - * @brief Handle partition-specific Fetch error. - */ -static void rd_kafka_fetch_reply_handle_partition_error( - rd_kafka_broker_t *rkb, - rd_kafka_toppar_t *rktp, - const struct rd_kafka_toppar_ver *tver, - rd_kafka_resp_err_t err, - int64_t HighwaterMarkOffset) { - - rd_rkb_dbg(rkb, FETCH, "FETCHERR", - "%.*s [%" PRId32 "]: Fetch failed at %s: %s", - RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), - rktp->rktp_partition, - rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos), - rd_kafka_err2name(err)); - - /* Some errors should be passed to the - * application while some handled by rdkafka */ - switch (err) { - /* Errors handled by rdkafka */ - case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART: - case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER: - case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR: - case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH: - case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH: - case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: - if (err == RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE) { - /* Occurs when: - * - Msg exists on broker but - * offset > HWM, or: - * - HWM is >= offset, but msg not - * yet available at that offset - * (replica is out of sync). - * - partition leader is out of sync. - * - * Handle by requesting metadata update, changing back - * to the leader, and then retrying FETCH - * (with backoff). - */ - rd_rkb_dbg(rkb, MSG, "FETCH", - "Topic %s [%" PRId32 - "]: %s not " - "available on broker %" PRId32 - " (leader %" PRId32 - "): updating metadata and retrying", - rktp->rktp_rkt->rkt_topic->str, - rktp->rktp_partition, - rd_kafka_fetch_pos2str( - rktp->rktp_offsets.fetch_pos), - rktp->rktp_broker_id, rktp->rktp_leader_id); - } - - if (err == RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH) { - rd_rkb_dbg(rkb, MSG | RD_KAFKA_DBG_CONSUMER, "FETCH", - "Topic %s [%" PRId32 - "]: Fetch failed at %s: %s: broker %" PRId32 - "has not yet caught up on latest metadata: " - "retrying", - rktp->rktp_rkt->rkt_topic->str, - rktp->rktp_partition, - rd_kafka_fetch_pos2str( - rktp->rktp_offsets.fetch_pos), - rd_kafka_err2str(err), rktp->rktp_broker_id); - } - - if (rktp->rktp_broker_id != rktp->rktp_leader_id) { - rd_kafka_toppar_delegate_to_leader(rktp); - } - /* Request metadata information update*/ - rd_kafka_toppar_leader_unavailable(rktp, "fetch", err); - break; - - case RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE: { - rd_kafka_fetch_pos_t err_pos; - - if (rktp->rktp_broker_id != rktp->rktp_leader_id && - rktp->rktp_offsets.fetch_pos.offset > HighwaterMarkOffset) { - rd_kafka_log(rkb->rkb_rk, LOG_WARNING, "FETCH", - "Topic %s [%" PRId32 - "]: %s " - " out of range (HighwaterMark %" PRId64 - " fetching from " - "broker %" PRId32 " (leader %" PRId32 - "): reverting to leader", - rktp->rktp_rkt->rkt_topic->str, - rktp->rktp_partition, - rd_kafka_fetch_pos2str( - rktp->rktp_offsets.fetch_pos), - HighwaterMarkOffset, rktp->rktp_broker_id, - rktp->rktp_leader_id); - - /* Out of range error cannot be taken as definitive - * when fetching from follower. - * Revert back to the leader in lieu of KIP-320. - */ - rd_kafka_toppar_delegate_to_leader(rktp); - break; - } - - /* Application error */ - err_pos = rktp->rktp_offsets.fetch_pos; - rktp->rktp_offsets.fetch_pos.offset = RD_KAFKA_OFFSET_INVALID; - rktp->rktp_offsets.fetch_pos.leader_epoch = -1; - rd_kafka_offset_reset(rktp, rd_kafka_broker_id(rkb), err_pos, - err, - "fetch failed due to requested offset " - "not available on the broker"); - } break; - - case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED: - /* If we're not authorized to access the - * topic mark it as errored to deny - * further Fetch requests. */ - if (rktp->rktp_last_error != err) { - rd_kafka_consumer_err( - rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err, - tver->version, NULL, rktp, - rktp->rktp_offsets.fetch_pos.offset, - "Fetch from broker %" PRId32 " failed: %s", - rd_kafka_broker_id(rkb), rd_kafka_err2str(err)); - rktp->rktp_last_error = err; - } - break; - - - /* Application errors */ - case RD_KAFKA_RESP_ERR__PARTITION_EOF: - if (rkb->rkb_rk->rk_conf.enable_partition_eof) - rd_kafka_consumer_err( - rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err, - tver->version, NULL, rktp, - rktp->rktp_offsets.fetch_pos.offset, - "Fetch from broker %" PRId32 - " reached end of " - "partition at offset %" PRId64 - " (HighwaterMark %" PRId64 ")", - rd_kafka_broker_id(rkb), - rktp->rktp_offsets.fetch_pos.offset, - HighwaterMarkOffset); - break; - - case RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE: - default: /* and all other errors */ - rd_dassert(tver->version > 0); - rd_kafka_consumer_err( - rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err, - tver->version, NULL, rktp, - rktp->rktp_offsets.fetch_pos.offset, - "Fetch from broker %" PRId32 " failed at %s: %s", - rd_kafka_broker_id(rkb), - rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos), - rd_kafka_err2str(err)); - break; - } - - /* Back off the next fetch for this partition */ - rd_kafka_toppar_fetch_backoff(rkb, rktp, err); -} - -static void rd_kafkap_Fetch_reply_tags_set_topic_cnt( - rd_kafkap_Fetch_reply_tags_t *reply_tags, - int32_t TopicCnt) { - reply_tags->TopicCnt = TopicCnt; - rd_dassert(!reply_tags->Topics); - reply_tags->Topics = rd_calloc(TopicCnt, sizeof(*reply_tags->Topics)); -} - -static void -rd_kafkap_Fetch_reply_tags_set_topic(rd_kafkap_Fetch_reply_tags_t *reply_tags, - int TopicIdx, - rd_kafka_Uuid_t TopicId, - int32_t PartitionCnt) { - reply_tags->Topics[TopicIdx].TopicId = TopicId; - reply_tags->Topics[TopicIdx].PartitionCnt = PartitionCnt; - rd_dassert(!reply_tags->Topics[TopicIdx].Partitions); - reply_tags->Topics[TopicIdx].Partitions = rd_calloc( - PartitionCnt, sizeof(*reply_tags->Topics[TopicIdx].Partitions)); -} - - -static void -rd_kafkap_Fetch_reply_tags_destroy(rd_kafkap_Fetch_reply_tags_t *reply_tags) { - int i; - for (i = 0; i < reply_tags->TopicCnt; i++) { - RD_IF_FREE(reply_tags->Topics[i].Partitions, rd_free); - } - RD_IF_FREE(reply_tags->Topics, rd_free); - RD_IF_FREE(reply_tags->NodeEndpoints.NodeEndpoints, rd_free); -} - -static int rd_kafkap_Fetch_reply_tags_partition_parse( - rd_kafka_buf_t *rkbuf, - uint64_t tagtype, - uint64_t taglen, - rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags, - rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) { - switch (tagtype) { - case 1: /* CurrentLeader */ - if (rd_kafka_buf_read_CurrentLeader( - rkbuf, &PartitionTags->CurrentLeader) == -1) - goto err_parse; - TopicTags->partitions_with_leader_change_cnt++; - return 1; - default: - return 0; - } -err_parse: - return -1; -} - -static int -rd_kafkap_Fetch_reply_tags_parse(rd_kafka_buf_t *rkbuf, - uint64_t tagtype, - uint64_t taglen, - rd_kafkap_Fetch_reply_tags_t *tags) { - switch (tagtype) { - case 0: /* NodeEndpoints */ - if (rd_kafka_buf_read_NodeEndpoints(rkbuf, - &tags->NodeEndpoints) == -1) - goto err_parse; - return 1; - default: - return 0; - } -err_parse: - return -1; -} - -static void -rd_kafka_handle_Fetch_metadata_update(rd_kafka_broker_t *rkb, - rd_kafkap_Fetch_reply_tags_t *FetchTags) { - if (FetchTags->topics_with_leader_change_cnt && - FetchTags->NodeEndpoints.NodeEndpoints) { - rd_kafka_metadata_t *md = NULL; - rd_kafka_metadata_internal_t *mdi = NULL; - rd_tmpabuf_t tbuf; - int32_t nodeid; - rd_kafka_op_t *rko; - int i, changed_topic, changed_partition; - - rd_kafka_broker_lock(rkb); - nodeid = rkb->rkb_nodeid; - rd_kafka_broker_unlock(rkb); - - rd_tmpabuf_new(&tbuf, 0, rd_true /*assert on fail*/); - rd_tmpabuf_add_alloc(&tbuf, sizeof(*mdi)); - rd_kafkap_leader_discovery_tmpabuf_add_alloc_brokers( - &tbuf, &FetchTags->NodeEndpoints); - rd_kafkap_leader_discovery_tmpabuf_add_alloc_topics( - &tbuf, FetchTags->topics_with_leader_change_cnt); - for (i = 0; i < FetchTags->TopicCnt; i++) { - if (!FetchTags->Topics[i] - .partitions_with_leader_change_cnt) - continue; - rd_kafkap_leader_discovery_tmpabuf_add_alloc_topic( - &tbuf, NULL, - FetchTags->Topics[i] - .partitions_with_leader_change_cnt); - } - rd_tmpabuf_finalize(&tbuf); - - mdi = rd_tmpabuf_alloc(&tbuf, sizeof(*mdi)); - md = &mdi->metadata; - - rd_kafkap_leader_discovery_metadata_init(mdi, nodeid); - - rd_kafkap_leader_discovery_set_brokers( - &tbuf, mdi, &FetchTags->NodeEndpoints); - - rd_kafkap_leader_discovery_set_topic_cnt( - &tbuf, mdi, FetchTags->topics_with_leader_change_cnt); - - changed_topic = 0; - for (i = 0; i < FetchTags->TopicCnt; i++) { - int j; - if (!FetchTags->Topics[i] - .partitions_with_leader_change_cnt) - continue; - - rd_kafkap_leader_discovery_set_topic( - &tbuf, mdi, changed_topic, - FetchTags->Topics[i].TopicId, NULL, - FetchTags->Topics[i] - .partitions_with_leader_change_cnt); - - changed_partition = 0; - for (j = 0; j < FetchTags->Topics[i].PartitionCnt; - j++) { - if (FetchTags->Topics[i] - .Partitions[j] - .CurrentLeader.LeaderId < 0) - continue; - - rd_kafkap_Fetch_reply_tags_Partition_t - *Partition = - &FetchTags->Topics[i].Partitions[j]; - rd_kafkap_leader_discovery_set_CurrentLeader( - &tbuf, mdi, changed_topic, - changed_partition, Partition->Partition, - &Partition->CurrentLeader); - changed_partition++; - } - changed_topic++; - } - - rko = rd_kafka_op_new(RD_KAFKA_OP_METADATA_UPDATE); - rko->rko_u.metadata.md = md; - rko->rko_u.metadata.mdi = mdi; - rd_kafka_q_enq(rkb->rkb_rk->rk_ops, rko); - } -} - -/** - * @brief Per-partition FetchResponse parsing and handling. - * - * @returns an error on buffer parse failure, else RD_KAFKA_RESP_ERR_NO_ERROR. - */ -static rd_kafka_resp_err_t rd_kafka_fetch_reply_handle_partition( - rd_kafka_broker_t *rkb, - const rd_kafkap_str_t *topic, - rd_kafka_topic_t *rkt /*possibly NULL*/, - rd_kafka_buf_t *rkbuf, - rd_kafka_buf_t *request, - int16_t ErrorCode, - rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags, - rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) { - const int log_decode_errors = LOG_ERR; - struct rd_kafka_toppar_ver *tver, tver_skel; - rd_kafka_toppar_t *rktp = NULL; - rd_kafka_aborted_txns_t *aborted_txns = NULL; - rd_slice_t save_slice; - int32_t fetch_version; - struct { - int32_t Partition; - int16_t ErrorCode; - int64_t HighwaterMarkOffset; - int64_t LastStableOffset; /* v4 */ - int64_t LogStartOffset; /* v5 */ - int32_t MessageSetSize; - int32_t PreferredReadReplica; /* v11 */ - } hdr; - rd_kafka_resp_err_t err; - int64_t end_offset; - - rd_kafka_buf_read_i32(rkbuf, &hdr.Partition); - rd_kafka_buf_read_i16(rkbuf, &hdr.ErrorCode); - if (PartitionTags) - PartitionTags->Partition = hdr.Partition; - if (ErrorCode) - hdr.ErrorCode = ErrorCode; - rd_kafka_buf_read_i64(rkbuf, &hdr.HighwaterMarkOffset); - - end_offset = hdr.HighwaterMarkOffset; - - hdr.LastStableOffset = RD_KAFKA_OFFSET_INVALID; - hdr.LogStartOffset = RD_KAFKA_OFFSET_INVALID; - if (rd_kafka_buf_ApiVersion(request) >= 4) { - int32_t AbortedTxnCnt; - int k; - rd_kafka_buf_read_i64(rkbuf, &hdr.LastStableOffset); - if (rd_kafka_buf_ApiVersion(request) >= 5) - rd_kafka_buf_read_i64(rkbuf, &hdr.LogStartOffset); - - rd_kafka_buf_read_arraycnt(rkbuf, &AbortedTxnCnt, - RD_KAFKAP_ABORTED_TRANSACTIONS_MAX); - - if (rkb->rkb_rk->rk_conf.isolation_level == - RD_KAFKA_READ_UNCOMMITTED) { - - if (unlikely(AbortedTxnCnt > 0)) { - rd_rkb_log(rkb, LOG_ERR, "FETCH", - "%.*s [%" PRId32 - "]: " - "%" PRId32 - " aborted transaction(s) " - "encountered in READ_UNCOMMITTED " - "fetch response: ignoring.", - RD_KAFKAP_STR_PR(topic), - hdr.Partition, AbortedTxnCnt); - for (k = 0; k < AbortedTxnCnt; k++) { - rd_kafka_buf_skip(rkbuf, (8 + 8)); - /* AbortedTransaction tags */ - rd_kafka_buf_skip_tags(rkbuf); - } - } - } else { - /* Older brokers may return LSO -1, - * in which case we use the HWM. */ - if (hdr.LastStableOffset >= 0) - end_offset = hdr.LastStableOffset; - - if (AbortedTxnCnt > 0) { - aborted_txns = - rd_kafka_aborted_txns_new(AbortedTxnCnt); - for (k = 0; k < AbortedTxnCnt; k++) { - int64_t PID; - int64_t FirstOffset; - rd_kafka_buf_read_i64(rkbuf, &PID); - rd_kafka_buf_read_i64(rkbuf, - &FirstOffset); - /* AbortedTransaction tags */ - rd_kafka_buf_skip_tags(rkbuf); - rd_kafka_aborted_txns_add( - aborted_txns, PID, FirstOffset); - } - rd_kafka_aborted_txns_sort(aborted_txns); - } - } - } - - if (rd_kafka_buf_ApiVersion(request) >= 11) - rd_kafka_buf_read_i32(rkbuf, &hdr.PreferredReadReplica); - else - hdr.PreferredReadReplica = -1; - /* Compact Records Array */ - rd_kafka_buf_read_arraycnt(rkbuf, &hdr.MessageSetSize, -1); - - if (unlikely(hdr.MessageSetSize < 0)) - rd_kafka_buf_parse_fail( - rkbuf, - "%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32, - RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize); - - /* Look up topic+partition */ - if (likely(rkt != NULL)) { - rd_kafka_topic_rdlock(rkt); - rktp = rd_kafka_toppar_get(rkt, hdr.Partition, - 0 /*no ua-on-miss*/); - rd_kafka_topic_rdunlock(rkt); - } - - if (unlikely(!rkt || !rktp)) { - rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC", - "Received Fetch response (error %hu) for unknown " - "topic %.*s [%" PRId32 "]: ignoring", - hdr.ErrorCode, RD_KAFKAP_STR_PR(topic), - hdr.Partition); - rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); - goto done; - } - - rd_kafka_toppar_lock(rktp); - rktp->rktp_lo_offset = hdr.LogStartOffset; - rktp->rktp_hi_offset = hdr.HighwaterMarkOffset; - /* Let the LastStable offset be the effective - * end_offset based on protocol version, that is: - * if connected to a broker that does not support - * LastStableOffset we use the HighwaterMarkOffset. */ - rktp->rktp_ls_offset = end_offset; - rd_kafka_toppar_unlock(rktp); - - if (hdr.PreferredReadReplica != -1) { - - rd_kafka_fetch_preferred_replica_handle( - rktp, rkbuf, rkb, hdr.PreferredReadReplica); - - if (unlikely(hdr.MessageSetSize != 0)) { - rd_rkb_log(rkb, LOG_WARNING, "FETCH", - "%.*s [%" PRId32 - "]: Fetch response has both preferred read " - "replica and non-zero message set size: " - "%" PRId32 ": skipping messages", - RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), - rktp->rktp_partition, hdr.MessageSetSize); - rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); - } - goto done; - } - - rd_kafka_toppar_lock(rktp); - - /* Make sure toppar hasn't moved to another broker - * during the lifetime of the request. */ - if (unlikely(rktp->rktp_broker != rkb)) { - rd_kafka_toppar_unlock(rktp); - rd_rkb_dbg(rkb, MSG, "FETCH", - "%.*s [%" PRId32 - "]: partition broker has changed: " - "discarding fetch response", - RD_KAFKAP_STR_PR(topic), hdr.Partition); - rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); - goto done; - } - - fetch_version = rktp->rktp_fetch_version; - rd_kafka_toppar_unlock(rktp); - - /* Check if this Fetch is for an outdated fetch version, - * or the original rktp was removed and a new one - * created (due to partition count decreasing and - * then increasing again, which can happen in - * desynchronized clusters): if so ignore it. */ - tver_skel.rktp = rktp; - tver = rd_list_find(request->rkbuf_rktp_vers, &tver_skel, - rd_kafka_toppar_ver_cmp); - rd_kafka_assert(NULL, tver); - if (tver->rktp != rktp || tver->version < fetch_version) { - rd_rkb_dbg(rkb, MSG, "DROP", - "%s [%" PRId32 - "]: dropping outdated fetch response " - "(v%d < %d or old rktp)", - rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, - tver->version, fetch_version); - rd_atomic64_add(&rktp->rktp_c.rx_ver_drops, 1); - rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); - goto done; - } - - rd_rkb_dbg(rkb, MSG, "FETCH", - "Topic %.*s [%" PRId32 "] MessageSet size %" PRId32 - ", error \"%s\", MaxOffset %" PRId64 ", LSO %" PRId64 - ", Ver %" PRId32 "/%" PRId32, - RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize, - rd_kafka_err2str(hdr.ErrorCode), hdr.HighwaterMarkOffset, - hdr.LastStableOffset, tver->version, fetch_version); - - /* If this is the last message of the queue, - * signal EOF back to the application. */ - if (end_offset == rktp->rktp_offsets.fetch_pos.offset && - rktp->rktp_offsets.eof_offset != end_offset) { - hdr.ErrorCode = RD_KAFKA_RESP_ERR__PARTITION_EOF; - rktp->rktp_offsets.eof_offset = end_offset; - } - - if (unlikely(hdr.ErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR)) { - /* Handle partition-level errors. */ - rd_kafka_fetch_reply_handle_partition_error( - rkb, rktp, tver, hdr.ErrorCode, hdr.HighwaterMarkOffset); - - rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); - goto done; - } - - /* No error, clear any previous fetch error. */ - rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR; - - if (unlikely(hdr.MessageSetSize <= 0)) - goto done; - - /** - * Parse MessageSet - */ - if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice, - (size_t)hdr.MessageSetSize)) - rd_kafka_buf_check_len(rkbuf, hdr.MessageSetSize); - - /* Parse messages */ - err = rd_kafka_msgset_parse(rkbuf, request, rktp, aborted_txns, tver); - - - rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice); - /* Continue with next partition regardless of - * parse errors (which are partition-specific) */ - - /* On error: back off the fetcher for this partition */ - if (unlikely(err)) - rd_kafka_toppar_fetch_backoff(rkb, rktp, err); - - goto done; - -err_parse: - if (aborted_txns) - rd_kafka_aborted_txns_destroy(aborted_txns); - if (rktp) - rd_kafka_toppar_destroy(rktp); /*from get()*/ - return rkbuf->rkbuf_err; - -done: - if (aborted_txns) - rd_kafka_aborted_txns_destroy(aborted_txns); - if (likely(rktp != NULL)) - rd_kafka_toppar_destroy(rktp); /*from get()*/ - - if (PartitionTags) { - /* Set default LeaderId and LeaderEpoch */ - PartitionTags->CurrentLeader.LeaderId = -1; - PartitionTags->CurrentLeader.LeaderEpoch = -1; - } - rd_kafka_buf_read_tags(rkbuf, - rd_kafkap_Fetch_reply_tags_partition_parse, - TopicTags, PartitionTags); - - return RD_KAFKA_RESP_ERR_NO_ERROR; -} - -/** - * Parses and handles a Fetch reply. - * Returns 0 on success or an error code on failure. - */ -static rd_kafka_resp_err_t -rd_kafka_fetch_reply_handle(rd_kafka_broker_t *rkb, - rd_kafka_buf_t *rkbuf, - rd_kafka_buf_t *request) { - int32_t TopicArrayCnt; - int i; - const int log_decode_errors = LOG_ERR; - rd_kafka_topic_t *rkt = NULL; - int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR; - rd_kafkap_Fetch_reply_tags_t FetchTags = RD_ZERO_INIT; - rd_bool_t has_fetch_tags = rd_false; - - if (rd_kafka_buf_ApiVersion(request) >= 1) { - int32_t Throttle_Time; - rd_kafka_buf_read_i32(rkbuf, &Throttle_Time); - - rd_kafka_op_throttle_time(rkb, rkb->rkb_rk->rk_rep, - Throttle_Time); - } - - if (rd_kafka_buf_ApiVersion(request) >= 7) { - int32_t SessionId; - rd_kafka_buf_read_i16(rkbuf, &ErrorCode); - rd_kafka_buf_read_i32(rkbuf, &SessionId); - } - - rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX); - /* Verify that TopicArrayCnt seems to be in line with remaining size */ - rd_kafka_buf_check_len(rkbuf, - TopicArrayCnt * (3 /*topic min size*/ + - 4 /*PartitionArrayCnt*/ + 4 + - 2 + 8 + 4 /*inner header*/)); - - if (rd_kafka_buf_ApiVersion(request) >= 12) { - has_fetch_tags = rd_true; - rd_kafkap_Fetch_reply_tags_set_topic_cnt(&FetchTags, - TopicArrayCnt); - } - - for (i = 0; i < TopicArrayCnt; i++) { - rd_kafkap_str_t topic = RD_ZERO_INIT; - rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO; - int32_t PartitionArrayCnt; - int j; - - if (rd_kafka_buf_ApiVersion(request) > 12) { - rd_kafka_buf_read_uuid(rkbuf, &topic_id); - rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, - topic_id); - if (rkt) - topic = *rkt->rkt_topic; - } else { - rd_kafka_buf_read_str(rkbuf, &topic); - rkt = rd_kafka_topic_find0(rkb->rkb_rk, &topic); - } - - rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt, - RD_KAFKAP_PARTITIONS_MAX); - if (rd_kafka_buf_ApiVersion(request) >= 12) { - rd_kafkap_Fetch_reply_tags_set_topic( - &FetchTags, i, topic_id, PartitionArrayCnt); - } - - for (j = 0; j < PartitionArrayCnt; j++) { - if (rd_kafka_fetch_reply_handle_partition( - rkb, &topic, rkt, rkbuf, request, ErrorCode, - has_fetch_tags ? &FetchTags.Topics[i] : NULL, - has_fetch_tags - ? &FetchTags.Topics[i].Partitions[j] - : NULL)) - goto err_parse; - } - if (has_fetch_tags && - FetchTags.Topics[i].partitions_with_leader_change_cnt) { - FetchTags.topics_with_leader_change_cnt++; - } - - if (rkt) { - rd_kafka_topic_destroy0(rkt); - rkt = NULL; - } - /* Topic Tags */ - rd_kafka_buf_skip_tags(rkbuf); - } - - /* Top level tags */ - rd_kafka_buf_read_tags(rkbuf, rd_kafkap_Fetch_reply_tags_parse, - &FetchTags); - - if (rd_kafka_buf_read_remain(rkbuf) != 0) { - rd_kafka_buf_parse_fail(rkbuf, - "Remaining data after message set " - "parse: %" PRIusz " bytes", - rd_kafka_buf_read_remain(rkbuf)); - RD_NOTREACHED(); - } - rd_kafka_handle_Fetch_metadata_update(rkb, &FetchTags); - rd_kafkap_Fetch_reply_tags_destroy(&FetchTags); - - return 0; - -err_parse: - if (rkt) - rd_kafka_topic_destroy0(rkt); - rd_kafkap_Fetch_reply_tags_destroy(&FetchTags); - rd_rkb_dbg(rkb, MSG, "BADMSG", - "Bad message (Fetch v%d): " - "is broker.version.fallback incorrectly set?", - (int)request->rkbuf_reqhdr.ApiVersion); - return rkbuf->rkbuf_err; -} - - - -/** - * @broker FetchResponse handling. - * - * @locality broker thread (or any thread if err == __DESTROY). - */ -static void rd_kafka_broker_fetch_reply(rd_kafka_t *rk, - rd_kafka_broker_t *rkb, - rd_kafka_resp_err_t err, - rd_kafka_buf_t *reply, - rd_kafka_buf_t *request, - void *opaque) { - - if (err == RD_KAFKA_RESP_ERR__DESTROY) - return; /* Terminating */ - - rd_kafka_assert(rkb->rkb_rk, rkb->rkb_fetching > 0); - rkb->rkb_fetching = 0; - - /* Parse and handle the messages (unless the request errored) */ - if (!err && reply) - err = rd_kafka_fetch_reply_handle(rkb, reply, request); - - if (unlikely(err)) { - char tmp[128]; - - rd_rkb_dbg(rkb, MSG, "FETCH", "Fetch reply: %s", - rd_kafka_err2str(err)); - switch (err) { - case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART: - case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION: - case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE: - case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: - /* Request metadata information update */ - rd_snprintf(tmp, sizeof(tmp), "FetchRequest failed: %s", - rd_kafka_err2str(err)); - rd_kafka_metadata_refresh_known_topics( - rkb->rkb_rk, NULL, rd_true /*force*/, tmp); - /* FALLTHRU */ - - case RD_KAFKA_RESP_ERR__TRANSPORT: - case RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT: - case RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: - /* The fetch is already intervalled from - * consumer_serve() so dont retry. */ - break; - - default: - break; - } - - rd_kafka_broker_fetch_backoff(rkb, err); - /* FALLTHRU */ - } -} - -/** - * @brief Check if any toppars have a zero topic id. - * - */ -static rd_bool_t can_use_topic_ids(rd_kafka_broker_t *rkb) { - rd_kafka_toppar_t *rktp = rkb->rkb_active_toppar_next; - do { - if (RD_KAFKA_UUID_IS_ZERO(rktp->rktp_rkt->rkt_topic_id)) - return rd_false; - } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp, - rktp_activelink)) != - rkb->rkb_active_toppar_next); - - return rd_true; -} - -/** - * @brief Build and send a Fetch request message for all underflowed toppars - * for a specific broker. - * - * @returns the number of partitions included in the FetchRequest, if any. - * - * @locality broker thread - */ -int rd_kafka_broker_fetch_toppars(rd_kafka_broker_t *rkb, rd_ts_t now) { - rd_kafka_toppar_t *rktp; - rd_kafka_buf_t *rkbuf; - int cnt = 0; - size_t of_TopicArrayCnt = 0; - int TopicArrayCnt = 0; - size_t of_PartitionArrayCnt = 0; - int PartitionArrayCnt = 0; - rd_kafka_topic_t *rkt_last = NULL; - int16_t ApiVersion = 0; - - /* Create buffer and segments: - * 1 x ReplicaId MaxWaitTime MinBytes TopicArrayCnt - * N x topic name - * N x PartitionArrayCnt Partition FetchOffset MaxBytes - * where N = number of toppars. - * Since we dont keep track of the number of topics served by - * this broker, only the partition count, we do a worst-case calc - * when allocating and assume each partition is on its own topic - */ - - if (unlikely(rkb->rkb_active_toppar_cnt == 0)) - return 0; - - ApiVersion = rd_kafka_broker_ApiVersion_supported(rkb, RD_KAFKAP_Fetch, - 0, 16, NULL); - - /* Fallback to version 12 if topic id is null which can happen if - * inter.broker.protocol.version is < 2.8 */ - if (ApiVersion > 12 && !can_use_topic_ids(rkb)) - ApiVersion = 12; - - rkbuf = rd_kafka_buf_new_flexver_request( - rkb, RD_KAFKAP_Fetch, 1, - /* MaxWaitTime+MinBytes+MaxBytes+IsolationLevel+ - * SessionId+Epoch+TopicCnt */ - 4 + 4 + 4 + 1 + 4 + 4 + 4 + - /* N x PartCnt+Partition+CurrentLeaderEpoch+FetchOffset+ - * LastFetchedEpoch+LogStartOffset+MaxBytes+?TopicNameLen?*/ - (rkb->rkb_active_toppar_cnt * - (4 + 4 + 4 + 8 + 4 + 8 + 4 + 40)) + - /* ForgottenTopicsCnt */ - 4 + - /* N x ForgottenTopicsData */ - 0, - ApiVersion >= 12); - - if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2) - rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, - RD_KAFKA_FEATURE_MSGVER2); - else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1) - rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, - RD_KAFKA_FEATURE_MSGVER1); - else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME) - rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, - RD_KAFKA_FEATURE_THROTTLETIME); - - - /* FetchRequest header */ - if (rd_kafka_buf_ApiVersion(rkbuf) <= 14) - /* ReplicaId */ - rd_kafka_buf_write_i32(rkbuf, -1); - - /* MaxWaitTime */ - rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_wait_max_ms); - /* MinBytes */ - rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_min_bytes); - - if (rd_kafka_buf_ApiVersion(rkbuf) >= 3) - /* MaxBytes */ - rd_kafka_buf_write_i32(rkbuf, - rkb->rkb_rk->rk_conf.fetch_max_bytes); - - if (rd_kafka_buf_ApiVersion(rkbuf) >= 4) - /* IsolationLevel */ - rd_kafka_buf_write_i8(rkbuf, - rkb->rkb_rk->rk_conf.isolation_level); - - if (rd_kafka_buf_ApiVersion(rkbuf) >= 7) { - /* SessionId */ - rd_kafka_buf_write_i32(rkbuf, 0); - /* Epoch */ - rd_kafka_buf_write_i32(rkbuf, -1); - } - - /* Write zero TopicArrayCnt but store pointer for later update */ - of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf); - - /* Prepare map for storing the fetch version for each partition, - * this will later be checked in Fetch response to purge outdated - * responses (e.g., after a seek). */ - rkbuf->rkbuf_rktp_vers = - rd_list_new(0, (void *)rd_kafka_toppar_ver_destroy); - rd_list_prealloc_elems(rkbuf->rkbuf_rktp_vers, - sizeof(struct rd_kafka_toppar_ver), - rkb->rkb_active_toppar_cnt, 0); - - /* Round-robin start of the list. */ - rktp = rkb->rkb_active_toppar_next; - do { - struct rd_kafka_toppar_ver *tver; - - if (rkt_last != rktp->rktp_rkt) { - if (rkt_last != NULL) { - /* Update PartitionArrayCnt */ - rd_kafka_buf_finalize_arraycnt( - rkbuf, of_PartitionArrayCnt, - PartitionArrayCnt); - /* Topic tags */ - rd_kafka_buf_write_tags_empty(rkbuf); - } - if (rd_kafka_buf_ApiVersion(rkbuf) > 12) { - /* Topic id must be non-zero here */ - rd_dassert(!RD_KAFKA_UUID_IS_ZERO( - rktp->rktp_rkt->rkt_topic_id)); - /* Topic ID */ - rd_kafka_buf_write_uuid( - rkbuf, &rktp->rktp_rkt->rkt_topic_id); - } else { - /* Topic name */ - rd_kafka_buf_write_kstr( - rkbuf, rktp->rktp_rkt->rkt_topic); - } - - TopicArrayCnt++; - rkt_last = rktp->rktp_rkt; - /* Partition count */ - of_PartitionArrayCnt = - rd_kafka_buf_write_arraycnt_pos(rkbuf); - PartitionArrayCnt = 0; - } - - PartitionArrayCnt++; - - /* Partition */ - rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition); - - if (rd_kafka_buf_ApiVersion(rkbuf) >= 9) { - /* CurrentLeaderEpoch */ - if (rktp->rktp_leader_epoch < 0 && - rd_kafka_has_reliable_leader_epochs(rkb)) { - /* If current leader epoch is set to -1 and - * the broker has reliable leader epochs, - * send 0 instead, so that epoch is checked - * and optionally metadata is refreshed. - * This can happen if metadata is read initially - * without an existing topic (see - * rd_kafka_topic_metadata_update2). - */ - rd_kafka_buf_write_i32(rkbuf, 0); - } else { - rd_kafka_buf_write_i32(rkbuf, - rktp->rktp_leader_epoch); - } - } - /* FetchOffset */ - rd_kafka_buf_write_i64(rkbuf, - rktp->rktp_offsets.fetch_pos.offset); - if (rd_kafka_buf_ApiVersion(rkbuf) >= 12) - /* LastFetchedEpoch - only used by follower replica */ - rd_kafka_buf_write_i32(rkbuf, -1); - if (rd_kafka_buf_ApiVersion(rkbuf) >= 5) - /* LogStartOffset - only used by follower replica */ - rd_kafka_buf_write_i64(rkbuf, -1); - - /* MaxBytes */ - rd_kafka_buf_write_i32(rkbuf, rktp->rktp_fetch_msg_max_bytes); - - /* Partition tags */ - rd_kafka_buf_write_tags_empty(rkbuf); - - rd_rkb_dbg(rkb, FETCH, "FETCH", - "Fetch topic %.*s [%" PRId32 "] at offset %" PRId64 - " (leader epoch %" PRId32 - ", current leader epoch %" PRId32 ", v%d)", - RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), - rktp->rktp_partition, - rktp->rktp_offsets.fetch_pos.offset, - rktp->rktp_offsets.fetch_pos.leader_epoch, - rktp->rktp_leader_epoch, rktp->rktp_fetch_version); - - /* We must have a valid fetch offset when we get here */ - rd_dassert(rktp->rktp_offsets.fetch_pos.offset >= 0); - - /* Add toppar + op version mapping. */ - tver = rd_list_add(rkbuf->rkbuf_rktp_vers, NULL); - tver->rktp = rd_kafka_toppar_keep(rktp); - tver->version = rktp->rktp_fetch_version; - - cnt++; - } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp, - rktp_activelink)) != - rkb->rkb_active_toppar_next); - - /* Update next toppar to fetch in round-robin list. */ - rd_kafka_broker_active_toppar_next( - rkb, rktp ? CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp, - rktp_activelink) - : NULL); - - rd_rkb_dbg(rkb, FETCH, "FETCH", "Fetch %i/%i/%i toppar(s)", cnt, - rkb->rkb_active_toppar_cnt, rkb->rkb_toppar_cnt); - if (!cnt) { - rd_kafka_buf_destroy(rkbuf); - return cnt; - } - - if (rkt_last != NULL) { - /* Update last topic's PartitionArrayCnt */ - rd_kafka_buf_finalize_arraycnt(rkbuf, of_PartitionArrayCnt, - PartitionArrayCnt); - /* Topic tags */ - rd_kafka_buf_write_tags_empty(rkbuf); - } - - /* Update TopicArrayCnt */ - rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt); - - - if (rd_kafka_buf_ApiVersion(rkbuf) >= 7) - /* Length of the ForgottenTopics list (KIP-227). Broker - * use only - not used by the consumer. */ - rd_kafka_buf_write_arraycnt(rkbuf, 0); - - if (rd_kafka_buf_ApiVersion(rkbuf) >= 11) - /* RackId */ - rd_kafka_buf_write_kstr(rkbuf, - rkb->rkb_rk->rk_conf.client_rack); - - /* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */ - if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000) - rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING; - - /* Use configured timeout */ - rd_kafka_buf_set_timeout(rkbuf, - rkb->rkb_rk->rk_conf.socket_timeout_ms + - rkb->rkb_rk->rk_conf.fetch_wait_max_ms, - now); - - /* Sort toppar versions for quicker lookups in Fetch response. */ - rd_list_sort(rkbuf->rkbuf_rktp_vers, rd_kafka_toppar_ver_cmp); - - rkb->rkb_fetching = 1; - rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_fetch_reply, NULL); - - return cnt; -} - -/** - * @brief Decide whether it should start fetching from next fetch start - * or continue with current fetch pos. - * - * @param rktp the toppar - * - * @returns rd_true if it should start fetching from next fetch start, - * rd_false otherwise. - * - * @locality any - * @locks toppar_lock() MUST be held - */ -static rd_bool_t rd_kafka_toppar_fetch_decide_start_from_next_fetch_start( - rd_kafka_toppar_t *rktp) { - return rktp->rktp_op_version > rktp->rktp_fetch_version || - rd_kafka_fetch_pos_cmp(&rktp->rktp_next_fetch_start, - &rktp->rktp_last_next_fetch_start) || - rktp->rktp_offsets.fetch_pos.offset == RD_KAFKA_OFFSET_INVALID; -} - -/** - * @brief Return next fetch start position: - * if it should start fetching from next fetch start - * or continue with current fetch pos. - * - * @param rktp The toppar - * - * @returns Next fetch start position - * - * @locality any - * @locks toppar_lock() MUST be held - */ -rd_kafka_fetch_pos_t -rd_kafka_toppar_fetch_decide_next_fetch_start_pos(rd_kafka_toppar_t *rktp) { - if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp)) - return rktp->rktp_next_fetch_start; - else - return rktp->rktp_offsets.fetch_pos; -} - -/** - * @brief Decide whether this toppar should be on the fetch list or not. - * - * Also: - * - update toppar's op version (for broker thread's copy) - * - finalize statistics (move rktp_offsets to rktp_offsets_fin) - * - * @returns the partition's Fetch backoff timestamp, or 0 if no backoff. - * - * @locality broker thread - * @locks none - */ -rd_ts_t rd_kafka_toppar_fetch_decide(rd_kafka_toppar_t *rktp, - rd_kafka_broker_t *rkb, - int force_remove) { - int should_fetch = 1; - const char *reason = ""; - int32_t version; - rd_ts_t ts_backoff = 0; - rd_bool_t lease_expired = rd_false; - - rd_kafka_toppar_lock(rktp); - - /* Check for preferred replica lease expiry */ - lease_expired = rktp->rktp_leader_id != rktp->rktp_broker_id && - rd_interval(&rktp->rktp_lease_intvl, - 5 * 60 * 1000 * 1000 /*5 minutes*/, 0) > 0; - if (lease_expired) { - /* delegate_to_leader() requires no locks to be held */ - rd_kafka_toppar_unlock(rktp); - rd_kafka_toppar_delegate_to_leader(rktp); - rd_kafka_toppar_lock(rktp); - - reason = "preferred replica lease expired"; - should_fetch = 0; - goto done; - } - - /* Forced removal from fetch list */ - if (unlikely(force_remove)) { - reason = "forced removal"; - should_fetch = 0; - goto done; - } - - if (unlikely((rktp->rktp_flags & RD_KAFKA_TOPPAR_F_REMOVE) != 0)) { - reason = "partition removed"; - should_fetch = 0; - goto done; - } - - /* Skip toppars not in active fetch state */ - if (rktp->rktp_fetch_state != RD_KAFKA_TOPPAR_FETCH_ACTIVE) { - reason = "not in active fetch state"; - should_fetch = 0; - goto done; - } - - /* Update broker thread's fetch op version */ - version = rktp->rktp_op_version; - if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp)) { - /* New version barrier, something was modified from the - * control plane. Reset and start over. - * Alternatively only the next_offset changed but not the - * barrier, which is the case when automatically triggering - * offset.reset (such as on PARTITION_EOF or - * OFFSET_OUT_OF_RANGE). */ - - rd_kafka_dbg( - rktp->rktp_rkt->rkt_rk, TOPIC, "FETCHDEC", - "Topic %s [%" PRId32 - "]: fetch decide: " - "updating to version %d (was %d) at %s " - "(was %s)", - rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, - version, rktp->rktp_fetch_version, - rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start), - rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos)); - - rd_kafka_offset_stats_reset(&rktp->rktp_offsets); - - /* New start offset */ - rktp->rktp_offsets.fetch_pos = rktp->rktp_next_fetch_start; - rktp->rktp_last_next_fetch_start = rktp->rktp_next_fetch_start; - - rktp->rktp_fetch_version = version; - - /* Clear last error to propagate new fetch - * errors if encountered. */ - rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR; - - rd_kafka_q_purge_toppar_version(rktp->rktp_fetchq, rktp, - version); - } - - - if (RD_KAFKA_TOPPAR_IS_PAUSED(rktp)) { - should_fetch = 0; - reason = "paused"; - - } else if (RD_KAFKA_OFFSET_IS_LOGICAL( - rktp->rktp_next_fetch_start.offset)) { - should_fetch = 0; - reason = "no concrete offset"; - } else if (rktp->rktp_ts_fetch_backoff > rd_clock()) { - reason = "fetch backed off"; - ts_backoff = rktp->rktp_ts_fetch_backoff; - should_fetch = 0; - } else if (rd_kafka_q_len(rktp->rktp_fetchq) >= - rkb->rkb_rk->rk_conf.queued_min_msgs) { - /* Skip toppars who's local message queue is already above - * the lower threshold. */ - reason = "queued.min.messages exceeded"; - ts_backoff = rd_kafka_toppar_fetch_backoff( - rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL); - should_fetch = 0; - - } else if ((int64_t)rd_kafka_q_size(rktp->rktp_fetchq) >= - rkb->rkb_rk->rk_conf.queued_max_msg_bytes) { - reason = "queued.max.messages.kbytes exceeded"; - ts_backoff = rd_kafka_toppar_fetch_backoff( - rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL); - should_fetch = 0; - } - -done: - /* Copy offset stats to finalized place holder. */ - rktp->rktp_offsets_fin = rktp->rktp_offsets; - - if (rktp->rktp_fetch != should_fetch) { - rd_rkb_dbg( - rkb, FETCH, "FETCH", - "Topic %s [%" PRId32 - "] in state %s at %s " - "(%d/%d msgs, %" PRId64 - "/%d kb queued, " - "opv %" PRId32 ") is %s%s", - rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, - rd_kafka_fetch_states[rktp->rktp_fetch_state], - rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start), - rd_kafka_q_len(rktp->rktp_fetchq), - rkb->rkb_rk->rk_conf.queued_min_msgs, - rd_kafka_q_size(rktp->rktp_fetchq) / 1024, - rkb->rkb_rk->rk_conf.queued_max_msg_kbytes, - rktp->rktp_fetch_version, - should_fetch ? "fetchable" : "not fetchable: ", reason); - - if (should_fetch) { - rd_dassert(rktp->rktp_fetch_version > 0); - rd_kafka_broker_active_toppar_add( - rkb, rktp, *reason ? reason : "fetchable"); - } else { - rd_kafka_broker_active_toppar_del(rkb, rktp, reason); - } - } - - rd_kafka_toppar_unlock(rktp); - - /* Non-fetching partitions will have an - * indefinate backoff, unless explicitly specified. */ - if (!should_fetch && !ts_backoff) - ts_backoff = RD_TS_MAX; - - return ts_backoff; -} diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer_oidc.c b/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer_oidc.c deleted file mode 100644 index d56efbf3554..00000000000 --- a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer_oidc.c +++ /dev/null @@ -1,589 +0,0 @@ -/* - * librdkafka - The Apache Kafka C/C++ library - * - * Copyright (c) 2021-2022, Magnus Edenhill - * 2023, Confluent Inc. - - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -/** - * Builtin SASL OAUTHBEARER OIDC support - */ -#include "rdkafka_int.h" -#include "rdkafka_sasl_int.h" -#include "rdunittest.h" -#include "cJSON.h" -#include -#include "rdhttp.h" -#include "rdkafka_sasl_oauthbearer_oidc.h" -#include "rdbase64.h" - - -/** - * @brief Generate Authorization field for HTTP header. - * The field contains base64-encoded string which - * is generated from \p client_id and \p client_secret. - * - * @returns Return the authorization field. - * - * @locality Any thread. - */ -static char *rd_kafka_oidc_build_auth_header(const char *client_id, - const char *client_secret) { - - rd_chariov_t client_authorization_in; - rd_chariov_t client_authorization_out; - - size_t authorization_base64_header_size; - char *authorization_base64_header; - - client_authorization_in.size = - strlen(client_id) + strlen(client_secret) + 2; - client_authorization_in.ptr = rd_malloc(client_authorization_in.size); - rd_snprintf(client_authorization_in.ptr, client_authorization_in.size, - "%s:%s", client_id, client_secret); - - client_authorization_in.size--; - rd_base64_encode(&client_authorization_in, &client_authorization_out); - rd_assert(client_authorization_out.ptr); - - authorization_base64_header_size = - strlen("Authorization: Basic ") + client_authorization_out.size + 1; - authorization_base64_header = - rd_malloc(authorization_base64_header_size); - rd_snprintf(authorization_base64_header, - authorization_base64_header_size, "Authorization: Basic %s", - client_authorization_out.ptr); - - rd_free(client_authorization_in.ptr); - rd_free(client_authorization_out.ptr); - return authorization_base64_header; -} - - -/** - * @brief Build headers for HTTP(S) requests based on \p client_id - * and \p client_secret. The result will be returned in \p *headersp. - * - * @locality Any thread. - */ -static void rd_kafka_oidc_build_headers(const char *client_id, - const char *client_secret, - struct curl_slist **headersp) { - char *authorization_base64_header; - - authorization_base64_header = - rd_kafka_oidc_build_auth_header(client_id, client_secret); - - *headersp = curl_slist_append(*headersp, "Accept: application/json"); - *headersp = curl_slist_append(*headersp, authorization_base64_header); - - *headersp = curl_slist_append( - *headersp, "Content-Type: application/x-www-form-urlencoded"); - - rd_free(authorization_base64_header); -} - -/** - * @brief The format of JWT is Header.Payload.Signature. - * Extract and decode payloads from JWT \p src. - * The decoded payloads will be returned in \p *bufplainp. - * - * @returns Return error message while decoding the payload. - */ -static const char *rd_kafka_jwt_b64_decode_payload(const char *src, - char **bufplainp) { - char *converted_src; - char *payload = NULL; - - const char *errstr = NULL; - - int i, padding, len; - - int payload_len; - int nbytesdecoded; - - int payloads_start = 0; - int payloads_end = 0; - - len = (int)strlen(src); - converted_src = rd_malloc(len + 4); - - for (i = 0; i < len; i++) { - switch (src[i]) { - case '-': - converted_src[i] = '+'; - break; - - case '_': - converted_src[i] = '/'; - break; - - case '.': - if (payloads_start == 0) - payloads_start = i + 1; - else { - if (payloads_end > 0) { - errstr = - "The token is invalid with more " - "than 2 delimiters"; - goto done; - } - payloads_end = i; - } - /* FALLTHRU */ - - default: - converted_src[i] = src[i]; - } - } - - if (payloads_start == 0 || payloads_end == 0) { - errstr = "The token is invalid with less than 2 delimiters"; - goto done; - } - - payload_len = payloads_end - payloads_start; - payload = rd_malloc(payload_len + 4); - strncpy(payload, (converted_src + payloads_start), payload_len); - - padding = 4 - (payload_len % 4); - if (padding < 4) { - while (padding--) - payload[payload_len++] = '='; - } - - nbytesdecoded = ((payload_len + 3) / 4) * 3; - *bufplainp = rd_malloc(nbytesdecoded + 1); - - if (EVP_DecodeBlock((uint8_t *)(*bufplainp), (uint8_t *)payload, - (int)payload_len) == -1) { - errstr = "Failed to decode base64 payload"; - } - -done: - RD_IF_FREE(payload, rd_free); - RD_IF_FREE(converted_src, rd_free); - return errstr; -} - -/** - * @brief Build post_fields with \p scope. - * The format of the post_fields is - * `grant_type=client_credentials&scope=scope` - * The post_fields will be returned in \p *post_fields. - * The post_fields_size will be returned in \p post_fields_size. - * - */ -static void rd_kafka_oidc_build_post_fields(const char *scope, - char **post_fields, - size_t *post_fields_size) { - size_t scope_size = 0; - - if (scope) - scope_size = strlen(scope); - if (scope_size == 0) { - *post_fields = rd_strdup("grant_type=client_credentials"); - *post_fields_size = strlen("grant_type=client_credentials"); - } else { - *post_fields_size = - strlen("grant_type=client_credentials&scope=") + scope_size; - *post_fields = rd_malloc(*post_fields_size + 1); - rd_snprintf(*post_fields, *post_fields_size + 1, - "grant_type=client_credentials&scope=%s", scope); - } -} - - -/** - * @brief Implementation of Oauth/OIDC token refresh callback function, - * will receive the JSON response after HTTP call to token provider, - * then extract the jwt from the JSON response, and forward it to - * the broker. - */ -void rd_kafka_oidc_token_refresh_cb(rd_kafka_t *rk, - const char *oauthbearer_config, - void *opaque) { - const int timeout_s = 20; - const int retry = 4; - const int retry_ms = 5 * 1000; - - double exp; - - cJSON *json = NULL; - cJSON *payloads = NULL; - cJSON *parsed_token, *jwt_exp, *jwt_sub; - - rd_http_error_t *herr; - - char *jwt_token; - char *post_fields; - char *decoded_payloads = NULL; - - struct curl_slist *headers = NULL; - - const char *token_url; - const char *sub; - const char *errstr; - - size_t post_fields_size; - size_t extension_cnt; - size_t extension_key_value_cnt = 0; - - char set_token_errstr[512]; - char decode_payload_errstr[512]; - - char **extensions = NULL; - char **extension_key_value = NULL; - - if (rd_kafka_terminating(rk)) - return; - - rd_kafka_oidc_build_headers(rk->rk_conf.sasl.oauthbearer.client_id, - rk->rk_conf.sasl.oauthbearer.client_secret, - &headers); - - /* Build post fields */ - rd_kafka_oidc_build_post_fields(rk->rk_conf.sasl.oauthbearer.scope, - &post_fields, &post_fields_size); - - token_url = rk->rk_conf.sasl.oauthbearer.token_endpoint_url; - - herr = rd_http_post_expect_json(rk, token_url, headers, post_fields, - post_fields_size, timeout_s, retry, - retry_ms, &json); - - if (unlikely(herr != NULL)) { - rd_kafka_log(rk, LOG_ERR, "OIDC", - "Failed to retrieve OIDC " - "token from \"%s\": %s (%d)", - token_url, herr->errstr, herr->code); - rd_kafka_oauthbearer_set_token_failure(rk, herr->errstr); - rd_http_error_destroy(herr); - goto done; - } - - parsed_token = cJSON_GetObjectItem(json, "access_token"); - - if (parsed_token == NULL) { - rd_kafka_oauthbearer_set_token_failure( - rk, - "Expected JSON JWT response with " - "\"access_token\" field"); - goto done; - } - - jwt_token = cJSON_GetStringValue(parsed_token); - if (jwt_token == NULL) { - rd_kafka_oauthbearer_set_token_failure( - rk, - "Expected JSON " - "response as a value string"); - goto done; - } - - errstr = rd_kafka_jwt_b64_decode_payload(jwt_token, &decoded_payloads); - if (errstr != NULL) { - rd_snprintf(decode_payload_errstr, - sizeof(decode_payload_errstr), - "Failed to decode JWT payload: %s", errstr); - rd_kafka_oauthbearer_set_token_failure(rk, - decode_payload_errstr); - goto done; - } - - payloads = cJSON_Parse(decoded_payloads); - if (payloads == NULL) { - rd_kafka_oauthbearer_set_token_failure( - rk, "Failed to parse JSON JWT payload"); - goto done; - } - - jwt_exp = cJSON_GetObjectItem(payloads, "exp"); - if (jwt_exp == NULL) { - rd_kafka_oauthbearer_set_token_failure( - rk, - "Expected JSON JWT response with " - "\"exp\" field"); - goto done; - } - - exp = cJSON_GetNumberValue(jwt_exp); - if (exp <= 0) { - rd_kafka_oauthbearer_set_token_failure( - rk, - "Expected JSON JWT response with " - "valid \"exp\" field"); - goto done; - } - - jwt_sub = cJSON_GetObjectItem(payloads, "sub"); - if (jwt_sub == NULL) { - rd_kafka_oauthbearer_set_token_failure( - rk, - "Expected JSON JWT response with " - "\"sub\" field"); - goto done; - } - - sub = cJSON_GetStringValue(jwt_sub); - if (sub == NULL) { - rd_kafka_oauthbearer_set_token_failure( - rk, - "Expected JSON JWT response with " - "valid \"sub\" field"); - goto done; - } - - if (rk->rk_conf.sasl.oauthbearer.extensions_str) { - extensions = - rd_string_split(rk->rk_conf.sasl.oauthbearer.extensions_str, - ',', rd_true, &extension_cnt); - - extension_key_value = rd_kafka_conf_kv_split( - (const char **)extensions, extension_cnt, - &extension_key_value_cnt); - } - - if (rd_kafka_oauthbearer_set_token( - rk, jwt_token, (int64_t)exp * 1000, sub, - (const char **)extension_key_value, extension_key_value_cnt, - set_token_errstr, - sizeof(set_token_errstr)) != RD_KAFKA_RESP_ERR_NO_ERROR) - rd_kafka_oauthbearer_set_token_failure(rk, set_token_errstr); - -done: - RD_IF_FREE(decoded_payloads, rd_free); - RD_IF_FREE(post_fields, rd_free); - RD_IF_FREE(json, cJSON_Delete); - RD_IF_FREE(headers, curl_slist_free_all); - RD_IF_FREE(extensions, rd_free); - RD_IF_FREE(extension_key_value, rd_free); - RD_IF_FREE(payloads, cJSON_Delete); -} - - -/** - * @brief Make sure the jwt is able to be extracted from HTTP(S) response. - * The JSON response after HTTP(S) call to token provider will be in - * rd_http_req_t.hreq_buf and jwt is the value of field "access_token", - * the format is {"access_token":"*******"}. - * This function mocks up the rd_http_req_t.hreq_buf using an dummy - * jwt. The rd_http_parse_json will extract the jwt from rd_http_req_t - * and make sure the extracted jwt is same with the dummy one. - */ -static int ut_sasl_oauthbearer_oidc_should_succeed(void) { - /* Generate a token in the https://jwt.io/ website by using the - * following steps: - * 1. Select the algorithm RS256 from the Algorithm drop-down menu. - * 2. Enter the header and the payload. - * payload should contains "exp", "iat", "sub", for example: - * payloads = {"exp": 1636532769, - "iat": 1516239022, - "sub": "sub"} - header should contains "kid", for example: - headers={"kid": "abcedfg"} */ - static const char *expected_jwt_token = - "eyJhbGciOiJIUzI1NiIsInR5" - "cCI6IkpXVCIsImtpZCI6ImFiY2VkZmcifQ" - "." - "eyJpYXQiOjE2MzIzNzUzMjAsInN1YiI6InN" - "1YiIsImV4cCI6MTYzMjM3NTYyMH0" - "." - "bT5oY8K-rS2gQ7Awc40844bK3zhzBhZb7sputErqQHY"; - char *expected_token_value; - size_t token_len; - rd_http_req_t hreq; - rd_http_error_t *herr; - cJSON *json = NULL; - char *token; - cJSON *parsed_token; - - RD_UT_BEGIN(); - - herr = rd_http_req_init(&hreq, ""); - - RD_UT_ASSERT(!herr, - "Expected initialize to succeed, " - "but failed with error code: %d, error string: %s", - herr->code, herr->errstr); - - token_len = strlen("access_token") + strlen(expected_jwt_token) + 8; - - expected_token_value = rd_malloc(token_len); - rd_snprintf(expected_token_value, token_len, "{\"%s\":\"%s\"}", - "access_token", expected_jwt_token); - rd_buf_write(hreq.hreq_buf, expected_token_value, token_len); - - herr = rd_http_parse_json(&hreq, &json); - RD_UT_ASSERT(!herr, - "Failed to parse JSON token: error code: %d, " - "error string: %s", - herr->code, herr->errstr); - - RD_UT_ASSERT(json, "Expected non-empty json."); - - parsed_token = cJSON_GetObjectItem(json, "access_token"); - - RD_UT_ASSERT(parsed_token, "Expected access_token in JSON response."); - token = parsed_token->valuestring; - - RD_UT_ASSERT(!strcmp(expected_jwt_token, token), - "Incorrect token received: " - "expected=%s; received=%s", - expected_jwt_token, token); - - rd_free(expected_token_value); - rd_http_error_destroy(herr); - rd_http_req_destroy(&hreq); - cJSON_Delete(json); - - RD_UT_PASS(); -} - - -/** - * @brief Make sure JSON doesn't include the "access_token" key, - * it will fail and return an empty token. - */ -static int ut_sasl_oauthbearer_oidc_with_empty_key(void) { - static const char *empty_token_format = "{}"; - size_t token_len; - rd_http_req_t hreq; - rd_http_error_t *herr; - cJSON *json = NULL; - cJSON *parsed_token; - - RD_UT_BEGIN(); - - herr = rd_http_req_init(&hreq, ""); - RD_UT_ASSERT(!herr, - "Expected initialization to succeed, " - "but it failed with error code: %d, error string: %s", - herr->code, herr->errstr); - - token_len = strlen(empty_token_format); - - rd_buf_write(hreq.hreq_buf, empty_token_format, token_len); - - herr = rd_http_parse_json(&hreq, &json); - - RD_UT_ASSERT(!herr, - "Expected JSON token parsing to succeed, " - "but it failed with error code: %d, error string: %s", - herr->code, herr->errstr); - - RD_UT_ASSERT(json, "Expected non-empty json."); - - parsed_token = cJSON_GetObjectItem(json, "access_token"); - - RD_UT_ASSERT(!parsed_token, - "Did not expecte access_token in JSON response"); - - rd_http_req_destroy(&hreq); - rd_http_error_destroy(herr); - cJSON_Delete(json); - cJSON_Delete(parsed_token); - RD_UT_PASS(); -} - -/** - * @brief Make sure the post_fields return correct with the scope. - */ -static int ut_sasl_oauthbearer_oidc_post_fields(void) { - static const char *scope = "test-scope"; - static const char *expected_post_fields = - "grant_type=client_credentials&scope=test-scope"; - - size_t expected_post_fields_size = strlen(expected_post_fields); - - size_t post_fields_size; - - char *post_fields; - - RD_UT_BEGIN(); - - rd_kafka_oidc_build_post_fields(scope, &post_fields, &post_fields_size); - - RD_UT_ASSERT(expected_post_fields_size == post_fields_size, - "Expected expected_post_fields_size is %" PRIusz - " received post_fields_size is %" PRIusz, - expected_post_fields_size, post_fields_size); - RD_UT_ASSERT(!strcmp(expected_post_fields, post_fields), - "Expected expected_post_fields is %s" - " received post_fields is %s", - expected_post_fields, post_fields); - - rd_free(post_fields); - - RD_UT_PASS(); -} - -/** - * @brief Make sure the post_fields return correct with the empty scope. - */ -static int ut_sasl_oauthbearer_oidc_post_fields_with_empty_scope(void) { - static const char *scope = NULL; - static const char *expected_post_fields = - "grant_type=client_credentials"; - - size_t expected_post_fields_size = strlen(expected_post_fields); - - size_t post_fields_size; - - char *post_fields; - - RD_UT_BEGIN(); - - rd_kafka_oidc_build_post_fields(scope, &post_fields, &post_fields_size); - - RD_UT_ASSERT(expected_post_fields_size == post_fields_size, - "Expected expected_post_fields_size is %" PRIusz - " received post_fields_size is %" PRIusz, - expected_post_fields_size, post_fields_size); - RD_UT_ASSERT(!strcmp(expected_post_fields, post_fields), - "Expected expected_post_fields is %s" - " received post_fields is %s", - expected_post_fields, post_fields); - - rd_free(post_fields); - - RD_UT_PASS(); -} - - -/** - * @brief make sure the jwt is able to be extracted from HTTP(S) requests - * or fail as expected. - */ -int unittest_sasl_oauthbearer_oidc(void) { - int fails = 0; - fails += ut_sasl_oauthbearer_oidc_should_succeed(); - fails += ut_sasl_oauthbearer_oidc_with_empty_key(); - fails += ut_sasl_oauthbearer_oidc_post_fields(); - fails += ut_sasl_oauthbearer_oidc_post_fields_with_empty_scope(); - return fails; -} diff --git a/lib/librdkafka-2.10.1/src/rdrand.c b/lib/librdkafka-2.10.1/src/rdrand.c deleted file mode 100644 index bdab002968e..00000000000 --- a/lib/librdkafka-2.10.1/src/rdrand.c +++ /dev/null @@ -1,70 +0,0 @@ -/* - * librd - Rapid Development C library - * - * Copyright (c) 2012-2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "rd.h" -#include "rdrand.h" -#include "rdtime.h" -#include "tinycthread.h" - -int rd_jitter(int low, int high) { - int rand_num; -#if HAVE_RAND_R - static RD_TLS unsigned int seed = 0; - - /* Initial seed with time+thread id */ - if (unlikely(seed == 0)) { - struct timeval tv; - rd_gettimeofday(&tv, NULL); - seed = (unsigned int)(tv.tv_usec / 1000); - seed ^= (unsigned int)(intptr_t)thrd_current(); - } - - rand_num = rand_r(&seed); -#else - rand_num = rand(); -#endif - return (low + (rand_num % ((high - low) + 1))); -} - -void rd_array_shuffle(void *base, size_t nmemb, size_t entry_size) { - int i; - void *tmp = rd_alloca(entry_size); - - /* FIXME: Optimized version for word-sized entries. */ - - for (i = (int)nmemb - 1; i > 0; i--) { - int j = rd_jitter(0, i); - if (unlikely(i == j)) - continue; - - memcpy(tmp, (char *)base + (i * entry_size), entry_size); - memcpy((char *)base + (i * entry_size), - (char *)base + (j * entry_size), entry_size); - memcpy((char *)base + (j * entry_size), tmp, entry_size); - } -} diff --git a/lib/librdkafka-2.10.1/tests/0009-mock_cluster.c b/lib/librdkafka-2.10.1/tests/0009-mock_cluster.c deleted file mode 100644 index a40fde2e2c0..00000000000 --- a/lib/librdkafka-2.10.1/tests/0009-mock_cluster.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2019-2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "test.h" - - -/** - * @name Verify that the builtin mock cluster works by producing to a topic - * and then consuming from it. - */ - - - -int main_0009_mock_cluster(int argc, char **argv) { - const char *topic = test_mk_topic_name("0009_mock_cluster", 1); - rd_kafka_mock_cluster_t *mcluster; - rd_kafka_t *p, *c; - rd_kafka_topic_t *rkt; - rd_kafka_conf_t *conf; - const int msgcnt = 100; - const char *bootstraps; - rd_kafka_topic_partition_list_t *parts; - - TEST_SKIP_MOCK_CLUSTER(0); - - mcluster = test_mock_cluster_new(3, &bootstraps); - - - test_conf_init(&conf, NULL, 30); - - test_conf_set(conf, "bootstrap.servers", bootstraps); - - /* Producer */ - rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); - p = test_create_handle(RD_KAFKA_PRODUCER, rd_kafka_conf_dup(conf)); - - /* Consumer */ - test_conf_set(conf, "auto.offset.reset", "earliest"); - c = test_create_consumer(topic, NULL, conf, NULL); - - rkt = test_create_producer_topic(p, topic, NULL); - test_wait_topic_exists(p, topic, 5000); - - /* Produce */ - test_produce_msgs(p, rkt, 0, RD_KAFKA_PARTITION_UA, 0, msgcnt, NULL, 0); - - /* Produce tiny messages */ - test_produce_msgs(p, rkt, 0, RD_KAFKA_PARTITION_UA, 0, msgcnt, "hello", - 5); - - rd_kafka_topic_destroy(rkt); - - /* Assign */ - parts = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add(parts, topic, 0); - rd_kafka_topic_partition_list_add(parts, topic, 1); - rd_kafka_topic_partition_list_add(parts, topic, 2); - rd_kafka_topic_partition_list_add(parts, topic, 3); - test_consumer_assign("CONSUME", c, parts); - rd_kafka_topic_partition_list_destroy(parts); - - - /* Consume */ - test_consumer_poll("CONSUME", c, 0, -1, 0, msgcnt, NULL); - - rd_kafka_destroy(c); - rd_kafka_destroy(p); - - test_mock_cluster_destroy(mcluster); - - return 0; -} diff --git a/lib/librdkafka-2.10.1/tests/0126-oauthbearer_oidc.c b/lib/librdkafka-2.10.1/tests/0126-oauthbearer_oidc.c deleted file mode 100644 index 7b889c9ee34..00000000000 --- a/lib/librdkafka-2.10.1/tests/0126-oauthbearer_oidc.c +++ /dev/null @@ -1,216 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2021-2022, Magnus Edenhill - * 2025, Confluent Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#include "test.h" -/* Typical include path would be , but this program - * is built from within the librdkafka source tree and thus differs. */ -#include "rdkafka.h" /* for Kafka driver */ - -static rd_bool_t error_seen; -/** - * @brief After config OIDC, make sure the producer and consumer - * can work successfully. - * - */ -static void -do_test_produce_consumer_with_OIDC(const rd_kafka_conf_t *base_conf) { - const char *topic; - uint64_t testid; - rd_kafka_t *p1; - rd_kafka_t *c1; - rd_kafka_conf_t *conf; - - const char *url = test_getenv("VALID_OIDC_URL", NULL); - - SUB_TEST("Test producer and consumer with oidc configuration"); - - if (!url) { - SUB_TEST_SKIP( - "VALID_OIDC_URL environment variable is not set\n"); - return; - } - - conf = rd_kafka_conf_dup(base_conf); - test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", url); - - testid = test_id_generate(); - - rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); - - p1 = test_create_handle(RD_KAFKA_PRODUCER, rd_kafka_conf_dup(conf)); - - topic = test_mk_topic_name("0126-oauthbearer_oidc", 1); - test_create_topic_wait_exists(p1, topic, 1, 3, 5000); - TEST_SAY("Topic: %s is created\n", topic); - - test_produce_msgs2(p1, topic, testid, 0, 0, 1, NULL, 0); - - test_conf_set(conf, "auto.offset.reset", "earliest"); - c1 = test_create_consumer(topic, NULL, rd_kafka_conf_dup(conf), NULL); - test_consumer_subscribe(c1, topic); - - /* Give it some time to trigger the token refresh. */ - rd_usleep(5 * 1000 * 1000, NULL); - test_consumer_poll("OIDC.C1", c1, testid, 1, -1, 1, NULL); - - test_consumer_close(c1); - - rd_kafka_destroy(p1); - rd_kafka_destroy(c1); - SUB_TEST_PASS(); -} - - -static void -auth_error_cb(rd_kafka_t *rk, int err, const char *reason, void *opaque) { - if (err == RD_KAFKA_RESP_ERR__AUTHENTICATION || - err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) { - TEST_SAY("Expected error: %s: %s\n", rd_kafka_err2str(err), - reason); - error_seen = rd_true; - } else - TEST_FAIL("Unexpected error: %s: %s", rd_kafka_err2str(err), - reason); - rd_kafka_yield(rk); -} - - -/** - * @brief After config OIDC, if the token is expired, make sure - * the authentication fail as expected. - * - */ -static void do_test_produce_consumer_with_OIDC_expired_token_should_fail( - const rd_kafka_conf_t *base_conf) { - rd_kafka_t *c1; - uint64_t testid; - rd_kafka_conf_t *conf; - - const char *expired_url = test_getenv("EXPIRED_TOKEN_OIDC_URL", NULL); - - SUB_TEST("Test OAUTHBEARER/OIDC failing with expired JWT"); - - if (!expired_url) { - SUB_TEST_SKIP( - "EXPIRED_TOKEN_OIDC_URL environment variable is not set\n"); - return; - } - - conf = rd_kafka_conf_dup(base_conf); - - error_seen = rd_false; - test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", expired_url); - - rd_kafka_conf_set_error_cb(conf, auth_error_cb); - - testid = test_id_generate(); - - c1 = test_create_consumer("OIDC.fail.C1", NULL, conf, NULL); - - test_consumer_poll_no_msgs("OIDC.fail.C1", c1, testid, 10 * 1000); - TEST_ASSERT(error_seen); - - test_consumer_close(c1); - rd_kafka_destroy(c1); - SUB_TEST_PASS(); -} - - -/** - * @brief After config OIDC, if the token is not valid, make sure the - * authentication fail as expected. - * - */ -static void do_test_produce_consumer_with_OIDC_should_fail( - const rd_kafka_conf_t *base_conf) { - rd_kafka_t *c1; - uint64_t testid; - rd_kafka_conf_t *conf; - - const char *invalid_url = test_getenv("INVALID_OIDC_URL", NULL); - - SUB_TEST("Test OAUTHBEARER/OIDC failing with invalid JWT"); - - if (!invalid_url) { - SUB_TEST_SKIP( - "INVALID_OIDC_URL environment variable is not set\n"); - return; - } - - conf = rd_kafka_conf_dup(base_conf); - - error_seen = rd_false; - - test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", invalid_url); - - rd_kafka_conf_set_error_cb(conf, auth_error_cb); - - testid = test_id_generate(); - - c1 = test_create_consumer("OIDC.fail.C1", NULL, conf, NULL); - - test_consumer_poll_no_msgs("OIDC.fail.C1", c1, testid, 10 * 1000); - - TEST_ASSERT(error_seen); - - test_consumer_close(c1); - rd_kafka_destroy(c1); - SUB_TEST_PASS(); -} - - -int main_0126_oauthbearer_oidc(int argc, char **argv) { - rd_kafka_conf_t *conf; - const char *sec; - const char *oidc; - - test_conf_init(&conf, NULL, 60); - - sec = test_conf_get(conf, "security.protocol"); - if (!strstr(sec, "sasl")) { - TEST_SKIP("Apache Kafka cluster not configured for SASL\n"); - rd_kafka_conf_destroy(conf); - return 0; - } - - oidc = test_conf_get(conf, "sasl.oauthbearer.method"); - if (rd_strcasecmp(oidc, "OIDC")) { - TEST_SKIP("`sasl.oauthbearer.method=OIDC` is required\n"); - rd_kafka_conf_destroy(conf); - return 0; - } - - do_test_produce_consumer_with_OIDC(conf); - do_test_produce_consumer_with_OIDC_should_fail(conf); - do_test_produce_consumer_with_OIDC_expired_token_should_fail(conf); - - rd_kafka_conf_destroy(conf); - - return 0; -} diff --git a/lib/librdkafka-2.10.1/tests/0135-sasl_credentials.cpp b/lib/librdkafka-2.10.1/tests/0135-sasl_credentials.cpp deleted file mode 100644 index 20e2e4f65c3..00000000000 --- a/lib/librdkafka-2.10.1/tests/0135-sasl_credentials.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * librdkafka - Apache Kafka C library - * - * Copyright (c) 2022, Magnus Edenhill - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -/** - * Verify that SASL credentials can be updated. - */ -#include "testcpp.h" - - - -class authErrorEventCb : public RdKafka::EventCb { - public: - authErrorEventCb() : error_seen(false) { - } - - void event_cb(RdKafka::Event &event) { - switch (event.type()) { - case RdKafka::Event::EVENT_ERROR: - Test::Say(tostr() << "Error: " << RdKafka::err2str(event.err()) << ": " - << event.str() << "\n"); - if (event.err() == RdKafka::ERR__AUTHENTICATION) - error_seen = true; - break; - - case RdKafka::Event::EVENT_LOG: - Test::Say(tostr() << "Log: " << event.str() << "\n"); - break; - - default: - break; - } - } - - bool error_seen; -}; - - -/** - * @brief Test setting SASL credentials. - * - * 1. Switch out the proper username/password for invalid ones. - * 2. Verify that we get an auth failure. - * 3. Set the proper username/passwords. - * 4. Verify that we can now connect. - */ -static void do_test(bool set_after_auth_failure) { - RdKafka::Conf *conf; - Test::conf_init(&conf, NULL, 30); - - SUB_TEST_QUICK("set_after_auth_failure=%s", - set_after_auth_failure ? "yes" : "no"); - - /* Get the correct sasl.username and sasl.password */ - std::string username, password; - if (conf->get("sasl.username", username) || - conf->get("sasl.password", password)) { - delete conf; - SUB_TEST_SKIP("sasl.username and/or sasl.password not configured\n"); - return; - } - - /* Replace with incorrect ones */ - Test::conf_set(conf, "sasl.username", "ThisIsNotRight"); - Test::conf_set(conf, "sasl.password", "Neither Is This"); - - /* Set up an event callback to track authentication errors */ - authErrorEventCb pEvent = authErrorEventCb(); - std::string errstr; - if (conf->set("event_cb", &pEvent, errstr) != RdKafka::Conf::CONF_OK) - Test::Fail(errstr); - - /* Create client */ - RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr); - if (!p) - Test::Fail("Failed to create Producer: " + errstr); - delete conf; - - if (set_after_auth_failure) { - Test::Say("Awaiting auth failure\n"); - - while (!pEvent.error_seen) - p->poll(1000); - - Test::Say("Authentication error seen\n"); - } - - Test::Say("Setting proper credentials\n"); - RdKafka::Error *error = p->sasl_set_credentials(username, password); - if (error) - Test::Fail("Failed to set credentials: " + error->str()); - - Test::Say("Expecting successful cluster authentication\n"); - const std::string clusterid = p->clusterid(5 * 1000); - - if (clusterid.empty()) - Test::Fail("Expected clusterid() to succeed"); - - delete p; - - SUB_TEST_PASS(); -} - -extern "C" { -int main_0135_sasl_credentials(int argc, char **argv) { - const char *mech = test_conf_get(NULL, "sasl.mechanism"); - - if (strcmp(mech, "PLAIN") && strncmp(mech, "SCRAM", 5)) { - Test::Skip("Test requires SASL PLAIN or SASL SCRAM\n"); - return 0; - } - - do_test(false); - do_test(true); - - return 0; -} -} diff --git a/lib/librdkafka-2.10.1/tests/requirements.txt b/lib/librdkafka-2.10.1/tests/requirements.txt deleted file mode 100644 index 523e5857ef8..00000000000 --- a/lib/librdkafka-2.10.1/tests/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -trivup/trivup-0.12.10.tar.gz -jsoncomment diff --git a/lib/librdkafka-2.10.1/win32/msbuild.ps1 b/lib/librdkafka-2.10.1/win32/msbuild.ps1 deleted file mode 100644 index 527d3e6661c..00000000000 --- a/lib/librdkafka-2.10.1/win32/msbuild.ps1 +++ /dev/null @@ -1,15 +0,0 @@ -param( - [string]$config='Release', - [string]$platform='x64', - [string]$toolset='v142' -) - -$msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -prerelease -products * -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe) - -echo "Using msbuild $msbuild" - -echo "Cleaning $config $platform $toolset" -& $msbuild win32\librdkafka.sln /p:Configuration=$config /p:Platform=$platform /p:PlatformToolset=$toolset /target:Clean - -echo "Building $config $platform $toolset" -& $msbuild win32\librdkafka.sln /p:Configuration=$config /p:Platform=$platform /p:PlatformToolset=$toolset diff --git a/lib/librdkafka-2.10.1/win32/setup-msys2.ps1 b/lib/librdkafka-2.10.1/win32/setup-msys2.ps1 deleted file mode 100644 index 052cc696441..00000000000 --- a/lib/librdkafka-2.10.1/win32/setup-msys2.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -# Install (if necessary) and set up msys2. - - -$url="https://github.com/msys2/msys2-installer/releases/download/2024-01-13/msys2-base-x86_64-20240113.sfx.exe" -$sha256="dba7e6d27e6a9ab850f502da44f6bfcd16d4d7b175fc2b25bee37207335cb12f" - - -if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) { - echo "Downloading and installing msys2 to c:\msys64" - - (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe') - - # Verify checksum - (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256 - - # Install msys2 - .\msys2-installer.exe -y -oc:\ - - Remove-Item msys2-installer.exe - - # Set up msys2 the first time - echo "Setting up msys" - c:\msys64\usr\bin\bash -lc ' ' - -} else { - echo "Using previously installed msys2" -} - -# Update packages -echo "Updating msys2 packages" -c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'" diff --git a/lib/librdkafka-2.10.1/win32/setup-vcpkg.ps1 b/lib/librdkafka-2.10.1/win32/setup-vcpkg.ps1 deleted file mode 100644 index 5fd1221a720..00000000000 --- a/lib/librdkafka-2.10.1/win32/setup-vcpkg.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -# Set up vcpkg and install required packages. - -if (!(Test-Path -Path vcpkg/.git)) { - git clone https://github.com/Microsoft/vcpkg.git -} - -cd vcpkg -git checkout 2024.09.30 -cd .. - -.\vcpkg\bootstrap-vcpkg.bat - diff --git a/lib/librdkafka-2.10.1/.clang-format-cpp b/lib/librdkafka-2.15.0/.clang-format-cpp similarity index 100% rename from lib/librdkafka-2.10.1/.clang-format-cpp rename to lib/librdkafka-2.15.0/.clang-format-cpp diff --git a/lib/librdkafka-2.10.1/.dir-locals.el b/lib/librdkafka-2.15.0/.dir-locals.el similarity index 100% rename from lib/librdkafka-2.10.1/.dir-locals.el rename to lib/librdkafka-2.15.0/.dir-locals.el diff --git a/lib/librdkafka-2.10.1/.formatignore b/lib/librdkafka-2.15.0/.formatignore similarity index 96% rename from lib/librdkafka-2.10.1/.formatignore rename to lib/librdkafka-2.15.0/.formatignore index ed5d1b43d28..141d3688ab5 100644 --- a/lib/librdkafka-2.10.1/.formatignore +++ b/lib/librdkafka-2.15.0/.formatignore @@ -3,6 +3,8 @@ # them in the original form to make future updates easier. src/lz4.c src/lz4.h +src/cJSON.c +src/cJSON.h src/lz4frame.c src/lz4frame.h src/lz4hc.c diff --git a/lib/librdkafka-2.10.1/.gdbmacros b/lib/librdkafka-2.15.0/.gdbmacros similarity index 100% rename from lib/librdkafka-2.10.1/.gdbmacros rename to lib/librdkafka-2.15.0/.gdbmacros diff --git a/lib/librdkafka-2.10.1/.github/CODEOWNERS b/lib/librdkafka-2.15.0/.github/CODEOWNERS similarity index 100% rename from lib/librdkafka-2.10.1/.github/CODEOWNERS rename to lib/librdkafka-2.15.0/.github/CODEOWNERS diff --git a/lib/librdkafka-2.10.1/.github/ISSUE_TEMPLATE b/lib/librdkafka-2.15.0/.github/ISSUE_TEMPLATE similarity index 100% rename from lib/librdkafka-2.10.1/.github/ISSUE_TEMPLATE rename to lib/librdkafka-2.15.0/.github/ISSUE_TEMPLATE diff --git a/lib/librdkafka-2.10.1/.gitignore b/lib/librdkafka-2.15.0/.gitignore similarity index 91% rename from lib/librdkafka-2.10.1/.gitignore rename to lib/librdkafka-2.15.0/.gitignore index 8e91b671100..76a4fcb904f 100644 --- a/lib/librdkafka-2.10.1/.gitignore +++ b/lib/librdkafka-2.15.0/.gitignore @@ -20,6 +20,7 @@ SOURCES gmon.out *.gz *.tgz +!tests/trivup/trivup*.tar.gz *.bz2 *.deb *.rpm diff --git a/lib/librdkafka-2.10.1/.semaphore/run-all-tests.yml b/lib/librdkafka-2.15.0/.semaphore/run-all-tests.yml similarity index 87% rename from lib/librdkafka-2.10.1/.semaphore/run-all-tests.yml rename to lib/librdkafka-2.15.0/.semaphore/run-all-tests.yml index d306651f479..d0af46ddaac 100644 --- a/lib/librdkafka-2.10.1/.semaphore/run-all-tests.yml +++ b/lib/librdkafka-2.15.0/.semaphore/run-all-tests.yml @@ -3,7 +3,7 @@ name: run-all-tests agent: machine: - type: s1-prod-ubuntu24-04-amd64-1 + type: s1-prod-ubuntu24-04-amd64-00 execution_time_limit: hours: 3 @@ -24,7 +24,8 @@ global_job_config: - ./configure --install-deps --enable-werror --enable-devel - make -j all - make -j -C tests build - - sem-version java 17 + - sem-version java 25 + - export JAVA_HOME="$(java -XshowSettings:properties -version 2>&1 | sed -n 's/^ *java.home = //p')" blocks: - name: "Run all tests (x86_64)" @@ -32,7 +33,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu24-04-amd64-2 + type: s1-prod-ubuntu24-04-amd64-1 prologue: commands: - if [[ "$TEST_ARCHES" != *"x86_64"* ]]; then exit 0; fi @@ -43,7 +44,7 @@ blocks: value: "False" commands: - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi - - ./tests/run-all-tests.sh + - ./tests/run-all-tests.sh - name: "SSL cluster (x86_64)" env_vars: - name: TEST_SSL @@ -56,7 +57,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu24-04-arm64-2 + type: s1-prod-ubuntu24-04-arm64-1 prologue: commands: - if [[ "$TEST_ARCHES" != *"aarch64"* ]]; then exit 0; fi diff --git a/lib/librdkafka-2.15.0/.semaphore/run-asan-tests.yml b/lib/librdkafka-2.15.0/.semaphore/run-asan-tests.yml new file mode 100644 index 00000000000..87152300163 --- /dev/null +++ b/lib/librdkafka-2.15.0/.semaphore/run-asan-tests.yml @@ -0,0 +1,53 @@ +version: v1.0 +name: run-share-consumer-tests-asan + +agent: + machine: + type: s1-prod-ubuntu24-04-amd64-00 + +execution_time_limit: + hours: 2 + +global_job_config: + prologue: + commands: + - checkout + - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF' + - sudo apt remove -y needrestart + - sudo DEBIAN_FRONTEND=noninteractive apt install -y libcurl4-openssl-dev libssl-dev + - python3 -m pip install -U pip + - (cd tests && python3 -m pip install -r requirements.txt) + - ./dev-conf.sh asan + - sem-version java 17 + - mkdir -p artifacts + env_vars: + - name: CACHE_TAG + value: '7' + - name: KAFKA_VERSION + value: '4.2.0' + +blocks: + - name: 'Share consumer tests (ASAN + LSAN)' + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + epilogue: + always: + commands: + - cp tests/asan-share-tests.log artifacts/ || true + - artifact push workflow artifacts/ --destination artifacts/asan/ + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} || true + jobs: + - name: 'Share consumer suite (ASAN + LSAN)' + env_vars: + - name: ASAN_OPTIONS + value: 'halt_on_error=0:abort_on_error=0:print_stacktrace=1:symbolize=1' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - (cd tests && python3 -m trivup.clusters.KafkaCluster + --kraft + --version "${KAFKA_VERSION}" + --conf '["group.share.min.record.lock.duration.ms=1000"]' + --cmd "bash -c 'set -o pipefail; TESTS_SKIP_BEFORE=0155 make 2>&1 | tee asan-share-tests.log'") diff --git a/lib/librdkafka-2.15.0/.semaphore/run-sasl-tests.yml b/lib/librdkafka-2.15.0/.semaphore/run-sasl-tests.yml new file mode 100644 index 00000000000..d7d5e042739 --- /dev/null +++ b/lib/librdkafka-2.15.0/.semaphore/run-sasl-tests.yml @@ -0,0 +1,105 @@ +version: v1.0 +name: run-share-consumer-tests-with-sasl + +agent: + machine: + type: s1-prod-ubuntu24-04-amd64-00 + +execution_time_limit: + hours: 3 + +global_job_config: + prologue: + commands: + - checkout + - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF' + - sudo apt remove -y needrestart + - sudo DEBIAN_FRONTEND=noninteractive apt install -y valgrind libcurl4-openssl-dev libssl-dev + - python3 -m pip install -U pip + - python3 -m pip -V + - (cd tests && python3 -m pip install -r requirements.txt) + - ./configure --install-deps --enable-werror --enable-devel + - make -j all + - make -j -C tests build + - sem-version java 17 + env_vars: + - name: CACHE_TAG + value: '7' + +blocks: + - name: "Run SASL tests (x86_64)" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + prologue: + commands: + - if [[ "$TEST_ARCHES" != *"x86_64"* ]]; then exit 0; fi + jobs: + - name: "SASL PLAINTEXT cluster (x86_64)" + env_vars: + - name: TEST_SSL + value: "False" + commands: + - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi + - 'export TEST_TRIVUP_PARAMETERS=''--conf ["connections.max.reauth.ms=10000"]''' + - 'if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then export TEST_TRIVUP_PARAMETERS=''--oidc --conf ["connections.max.reauth.ms=10000"]''; fi' + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache restore trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG}; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then (cd tests && echo "TESTS=0126 make 2>&1; exit" | python3 interactive_broker_version.py --kraft --sasl OAUTHBEARER --oauthbearer-method OIDC $([ "$TEST_SSL" = "True" ] && echo --ssl) ${TEST_KAFKA_GIT_REF:-4.3.0}) 2>&1 | tee /tmp/0126.log; if grep -qE "TEST FAILURE|timed out|test-runner.*FAILED|run_par.*Error" /tmp/0126.log; then echo "OIDC tests (0126) FAILED"; exit 1; else echo "OIDC tests (0126) PASSED"; fi; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache store trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp/LibrdkafkaTestCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0}; fi + - cache restore trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} + - export TESTS=0022,0128,0135,0142 && ./tests/run-all-tests.sh + - cache store trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0} + - name: "SASL SSL cluster (x86_64)" + env_vars: + - name: TEST_SSL + value: "True" + commands: + - if [[ "$TEST_TYPE" != *"ssl"* ]]; then exit 0; fi + - 'export TEST_TRIVUP_PARAMETERS=''--conf ["connections.max.reauth.ms=10000"]''' + - 'if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then export TEST_TRIVUP_PARAMETERS=''--oidc --conf ["connections.max.reauth.ms=10000"]''; fi' + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache restore trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG}; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then (cd tests && echo "TESTS=0126 make 2>&1; exit" | python3 interactive_broker_version.py --kraft --sasl OAUTHBEARER --oauthbearer-method OIDC $([ "$TEST_SSL" = "True" ] && echo --ssl) ${TEST_KAFKA_GIT_REF:-4.3.0}) 2>&1 | tee /tmp/0126.log; if grep -qE "TEST FAILURE|timed out|test-runner.*FAILED|run_par.*Error" /tmp/0126.log; then echo "OIDC tests (0126) FAILED"; exit 1; else echo "OIDC tests (0126) PASSED"; fi; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache store trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp/LibrdkafkaTestCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0}; fi + - cache restore trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} + - export TESTS=0022,0128,0135,0142 && ./tests/run-all-tests.sh + - cache store trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0} + - name: "Run SASL tests (aarch64)" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-1 + prologue: + commands: + - if [[ "$TEST_ARCHES" != *"aarch64"* ]]; then exit 0; fi + jobs: + - name: "SASL PLAINTEXT cluster (aarch64)" + env_vars: + - name: TEST_SSL + value: "False" + commands: + - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi + - 'export TEST_TRIVUP_PARAMETERS=''--conf ["connections.max.reauth.ms=10000"]''' + - 'if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then export TEST_TRIVUP_PARAMETERS=''--oidc --conf ["connections.max.reauth.ms=10000"]''; fi' + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache restore trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG}; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then (cd tests && echo "TESTS=0126 make 2>&1; exit" | python3 interactive_broker_version.py --kraft --sasl OAUTHBEARER --oauthbearer-method OIDC $([ "$TEST_SSL" = "True" ] && echo --ssl) ${TEST_KAFKA_GIT_REF:-4.3.0}) 2>&1 | tee /tmp/0126.log; if grep -qE "TEST FAILURE|timed out|test-runner.*FAILED|run_par.*Error" /tmp/0126.log; then echo "OIDC tests (0126) FAILED"; exit 1; else echo "OIDC tests (0126) PASSED"; fi; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache store trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp/LibrdkafkaTestCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0}; fi + - cache restore trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} + - export TESTS=0022,0128,0135,0142 && ./tests/run-all-tests.sh + - cache store trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0} + - name: "SASL SSL cluster (aarch64)" + env_vars: + - name: TEST_SSL + value: "True" + commands: + - if [[ "$TEST_TYPE" != *"ssl"* ]]; then exit 0; fi + - 'export TEST_TRIVUP_PARAMETERS=''--conf ["connections.max.reauth.ms=10000"]''' + - 'if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then export TEST_TRIVUP_PARAMETERS=''--oidc --conf ["connections.max.reauth.ms=10000"]''; fi' + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache restore trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG}; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then (cd tests && echo "TESTS=0126 make 2>&1; exit" | python3 interactive_broker_version.py --kraft --sasl OAUTHBEARER --oauthbearer-method OIDC $([ "$TEST_SSL" = "True" ] && echo --ssl) ${TEST_KAFKA_GIT_REF:-4.3.0}) 2>&1 | tee /tmp/0126.log; if grep -qE "TEST FAILURE|timed out|test-runner.*FAILED|run_par.*Error" /tmp/0126.log; then echo "OIDC tests (0126) FAILED"; exit 1; else echo "OIDC tests (0126) PASSED"; fi; fi + - if [[ "$TEST_SASL" == "OAUTHBEARER" ]]; then cache store trivup-kafka-oidc-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp/LibrdkafkaTestCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0}; fi + - cache restore trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} + - export TESTS=0022,0128,0135,0142 && ./tests/run-all-tests.sh + - cache store trivup-kafka-${TEST_KAFKA_GIT_REF:-4.3.0}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${TEST_KAFKA_GIT_REF:-4.3.0} diff --git a/lib/librdkafka-2.15.0/.semaphore/run-tsan-tests.yml b/lib/librdkafka-2.15.0/.semaphore/run-tsan-tests.yml new file mode 100644 index 00000000000..1634efb4d45 --- /dev/null +++ b/lib/librdkafka-2.15.0/.semaphore/run-tsan-tests.yml @@ -0,0 +1,54 @@ +version: v1.0 +name: run-share-consumer-tests-tsan + +agent: + machine: + type: s1-prod-ubuntu24-04-amd64-00 + +execution_time_limit: + hours: 2 + +global_job_config: + prologue: + commands: + - checkout + - sudo sysctl -w vm.mmap_rnd_bits=28 + - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF' + - sudo apt remove -y needrestart + - sudo DEBIAN_FRONTEND=noninteractive apt install -y libcurl4-openssl-dev libssl-dev + - python3 -m pip install -U pip + - (cd tests && python3 -m pip install -r requirements.txt) + - ./dev-conf.sh tsan + - sem-version java 17 + - mkdir -p artifacts + env_vars: + - name: CACHE_TAG + value: '7' + - name: KAFKA_VERSION + value: '4.2.0' + +blocks: + - name: 'Share consumer tests (TSAN)' + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + epilogue: + always: + commands: + - cp tests/tsan-share-tests.log artifacts/ || true + - artifact push workflow artifacts/ --destination artifacts/tsan/ + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} || true + jobs: + - name: 'Share consumer suite (TSAN)' + env_vars: + - name: TSAN_OPTIONS + value: 'halt_on_error=0:history_size=7:second_deadlock_stack=1:symbolize=1' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - (cd tests && python3 -m trivup.clusters.KafkaCluster + --kraft + --version "${KAFKA_VERSION}" + --conf '["group.share.min.record.lock.duration.ms=1000"]' + --cmd "bash -c 'set -o pipefail; TESTS_SKIP_BEFORE=0155 make 2>&1 | tee tsan-share-tests.log'") diff --git a/lib/librdkafka-2.15.0/.semaphore/run-valgrind-tests.yml b/lib/librdkafka-2.15.0/.semaphore/run-valgrind-tests.yml new file mode 100644 index 00000000000..29df75e2cc4 --- /dev/null +++ b/lib/librdkafka-2.15.0/.semaphore/run-valgrind-tests.yml @@ -0,0 +1,52 @@ +version: v1.0 +name: run-share-consumer-tests-valgrind + +agent: + machine: + type: s1-prod-ubuntu24-04-amd64-00 + +execution_time_limit: + hours: 2 + +global_job_config: + prologue: + commands: + - checkout + - mkdir -p artifacts + - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF' + - sudo apt remove -y needrestart + - sudo DEBIAN_FRONTEND=noninteractive apt install -y valgrind libcurl4-openssl-dev libssl-dev + - python3 -m pip install -U pip + - (cd tests && python3 -m pip install -r requirements.txt) + - ./configure --install-deps --enable-devel --disable-optimization + - make -j all + - make -j -C tests build + - sem-version java 17 + env_vars: + - name: CACHE_TAG + value: '7' + - name: KAFKA_VERSION + value: '4.2.0' + +blocks: + - name: 'Share consumer tests (Valgrind memcheck)' + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + epilogue: + always: + commands: + - cp tests/valgrind-share-tests.log artifacts/ || true + - artifact push workflow artifacts/ --destination artifacts/valgrind/ + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} || true + jobs: + - name: 'Share consumer suite (Valgrind memcheck)' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - (cd tests && python3 -m trivup.clusters.KafkaCluster + --kraft + --version "${KAFKA_VERSION}" + --conf '["group.share.min.record.lock.duration.ms=1000"]' + --cmd "bash -c 'set -o pipefail; TESTS_SKIP_BEFORE=0155 ./run-test.sh valgrind 2>&1 | tee valgrind-share-tests.log'") diff --git a/lib/librdkafka-2.15.0/.semaphore/semaphore-integration.yml b/lib/librdkafka-2.15.0/.semaphore/semaphore-integration.yml new file mode 100644 index 00000000000..a4d4a8bbfba --- /dev/null +++ b/lib/librdkafka-2.15.0/.semaphore/semaphore-integration.yml @@ -0,0 +1,236 @@ +version: v1.0 +name: librdkafka-integration-tests +agent: + machine: + type: s1-prod-ubuntu24-04-arm64-00 +execution_time_limit: + hours: 3 + +auto_cancel: + running: + when: "branch != 'master'" + +global_job_config: + prologue: + commands: + - checkout + - mkdir artifacts + - mkdir dest + env_vars: + - name: CACHE_TAG + value: '7' + - name: CI + value: 'true' + - name: KAFKA_VERSION + value: '4.3.0' + - name: CP_VERSION + value: '8.0.0' + # Shows plain output from docker build (no progress bars) + - name: BUILDKIT_PROGRESS + value: plain + - name: MACOSX_DEPLOYMENT_TARGET + value: '13' + +blocks: + - name: 'OSX arm64/m1' + dependencies: [] + task: + agent: + machine: + type: s1-macos-15-arm64-8 + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-osx__arch-arm64__lnk-all + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build' + commands: + - packaging/macos/build-release-artifacts.sh + + + - name: 'OSX x64' + dependencies: [] + task: + agent: + machine: + type: s1-macos-15-amd64-6 + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-osx__arch-x64__lnk-all + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build' + commands: + - packaging/macos/build-release-artifacts.sh + + - name: 'Linux Ubuntu amd64: integration tests' + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + env_vars: + - name: CFLAGS + value: -std=gnu90 # Test minimum C standard, default in CentOS 7 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + jobs: + - name: 'Build and integration tests with "classic" protocol' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG} + ./packaging/tools/run-integration-tests.sh + ${KAFKA_VERSION} ${CP_VERSION} classic + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} + - name: 'Build and integration tests with "consumer" protocol' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG} + ./packaging/tools/run-integration-tests.sh + ${KAFKA_VERSION} ${CP_VERSION} consumer + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} + + + - name: 'Linux arm64: local quick tests' + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-1 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build: centos8 glibc +gssapi' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi + commands: + - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Build: centos8 glibc' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all + commands: + - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Build: alpine musl +gssapi' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi + commands: + - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz + + - name: 'Build: alpine musl' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all + commands: + - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz + + + - name: 'Windows x64: MinGW-w64' + dependencies: [] + task: + agent: + machine: + type: s1-prod-windows + env_vars: + - name: CHERE_INVOKING + value: 'yes' + - name: MSYSTEM + value: UCRT64 + prologue: + commands: + # Set up msys2 + - "& .\\win32\\setup-msys2.ps1" + epilogue: + commands: + - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ } + jobs: + - name: 'Build: MinGW-w64 Dynamic' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std + commands: + - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh ./artifacts/librdkafka.tgz' + + - name: 'Build: MinGW-w64 Static' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static + commands: + - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh --static ./artifacts/librdkafka.tgz' + + - name: 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019' + dependencies: [] + task: + agent: + machine: + type: s1-prod-windows + env_vars: + # Disable vcpkg telemetry + - name: VCPKG_DISABLE_METRICS + value: 'yes' + prologue: + commands: + # install vcpkg in the parent directory. + - cd .. + # Setup vcpkg + - "& .\\librdkafka\\win32\\setup-vcpkg.ps1" + - cd librdkafka + epilogue: + commands: + - Get-ChildItem . -include *.dll -recurse + - Get-ChildItem . -include *.lib -recurse + - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ } + jobs: + - name: 'Build: MSVC x64' + env_vars: + - name: triplet + value: x64-windows + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std + commands: + - "& .\\win32\\msbuild.ps1 -platform x64" + - "& .\\win32\\package-zip.ps1 -platform x64" + - name: 'Build: MSVC x86' + env_vars: + - name: triplet + value: x86-windows + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std + commands: + - "& .\\win32\\msbuild.ps1 -platform Win32" + - "& .\\win32\\package-zip.ps1 -platform Win32" + +promotions: + - name: Run all tests on master commits + pipeline_file: run-all-tests.yml + parameters: + env_vars: + - required: true + name: TEST_KAFKA_GIT_REF + default_value: 4.3.0 + - required: true + name: TEST_TYPE + default_value: plaintext,ssl + - required: true + name: TEST_ARCHES + default_value: x86_64,aarch64 + - required: true + name: TEST_PARALLEL + default_value: "1" + auto_promote_on: + - result: passed + branch: + - "master" diff --git a/lib/librdkafka-2.15.0/.semaphore/semaphore.yml b/lib/librdkafka-2.15.0/.semaphore/semaphore.yml new file mode 100644 index 00000000000..3fb2d0a4023 --- /dev/null +++ b/lib/librdkafka-2.15.0/.semaphore/semaphore.yml @@ -0,0 +1,631 @@ +version: v1.0 +name: build-test-release +agent: + machine: + type: s1-prod-ubuntu24-04-arm64-00 +execution_time_limit: + hours: 3 + +auto_cancel: + running: + when: "branch != 'master'" + +global_job_config: + prologue: + commands: + - checkout + - mkdir artifacts + - mkdir dest + env_vars: + - name: CACHE_TAG + value: '8' + - name: CI + value: 'true' + - name: KAFKA_VERSION + value: '4.3.0' + - name: CP_VERSION + value: '8.0.0' + # Shows plain output from docker build (no progress bars) + - name: BUILDKIT_PROGRESS + value: plain + - name: MACOSX_DEPLOYMENT_TARGET + value: '13' + +blocks: + - name: 'OSX arm64/m1' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-macos-15-arm64-8 + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-osx__arch-arm64__lnk-all + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build' + commands: + - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip + - make -j all examples check + - examples/rdkafka_example -X builtin.features + - otool -L src/librdkafka.dylib + - otool -L src-cpp/librdkafka++.dylib + - make -j -C tests build + - make -C tests run_local_quick + - DESTDIR="$PWD/dest" make install + - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .) + + + - name: 'OSX x64' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-macos-15-amd64-6 + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-osx__arch-x64__lnk-all + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build' + commands: + - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip + - make -j all examples check + - examples/rdkafka_example -X builtin.features + - otool -L src/librdkafka.dylib + - otool -L src-cpp/librdkafka++.dylib + - make -j -C tests build + - make -C tests run_local_quick + - DESTDIR="$PWD/dest" make install + - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .) + + + - name: 'Style check' + dependencies: [] + skip: + # Skip for release tags, we don't want style checks + # to fail the release build. + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-00 + jobs: + - name: 'Style check' + commands: + - packaging/tools/run-in-docker.sh test-runner-arm64-${CACHE_TAG} packaging/tools/run-style-check.sh + + - name: 'Build documentation' + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-1 + jobs: + - name: 'Generate documentation' + commands: + - packaging/tools/run-in-docker.sh test-runner-arm64-${CACHE_TAG} make docs + - (cd staging-docs && tar cvzf ../artifacts/librdkafka-docs.tgz .) + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/librdkafka-docs.tgz --destination artifacts/librdkafka-docs.tgz' + + + - name: 'Linux Ubuntu amd64: source build' + dependencies: [] + skip: + # Skip for release tags, we don't want flaky CI tests + # to fail the release build. + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + env_vars: + - name: CFLAGS + value: -std=gnu90 # Test minimum C standard, default in CentOS 7 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + jobs: + - name: 'Build configuration checks' + commands: + - python3 -m pip install -U pip + - ./packaging/tools/build-configurations-checks.sh make test-runner-manylinux-amd64-${CACHE_TAG} + - ./packaging/tools/build-configurations-checks.sh cmake test-runner-manylinux-amd64-${CACHE_TAG} + - ./packaging/tools/build-configurations-checks.sh make test-runner-alpine-amd64-${CACHE_TAG} + - ./packaging/tools/build-configurations-checks.sh cmake test-runner-alpine-amd64-${CACHE_TAG} + + - name: 'Linux Ubuntu amd64: share consumer tests' + dependencies: [] + skip: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + jobs: + - name: 'Linux Ubuntu amd64: share consumer tests' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG} + ./packaging/tools/run-share-consumer-tests.sh + ${KAFKA_VERSION} ${CP_VERSION} + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} + + - name: 'Linux Ubuntu amd64: share consumer telemetry' + dependencies: [] + skip: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + - sem-version java 17 + epilogue: + always: + commands: + - 'ls artifacts/ 2>/dev/null && artifact push job artifacts/ --destination telemetry-debug-${SEMAPHORE_JOB_ID}/ || true' + jobs: + - name: 'Linux Ubuntu amd64: share consumer telemetry' + commands: + - cache restore share-telemetry-${KAFKA_VERSION}-${CACHE_TAG} + - ./packaging/tools/setup-share-telemetry-testing.sh + - cache store share-telemetry-${KAFKA_VERSION}-${CACHE_TAG} tests/share_telemetry_verify/.work + + - name: 'Linux Ubuntu aarch64: share consumer tests' + dependencies: [] + skip: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-1 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + jobs: + - name: 'Linux Ubuntu aarch64: share consumer tests' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - packaging/tools/run-in-docker.sh test-runner-arm64-${CACHE_TAG} + ./packaging/tools/run-share-consumer-tests.sh + ${KAFKA_VERSION} ${CP_VERSION} + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} + +# TODO KIP-932: Re-enable once M1 issues on share consumer tests are fixed. +# - name: 'OSX arm64/m1: share consumer tests' +# dependencies: [] +# skip: +# when: "tag =~ '^v[0-9]\\.'" +# task: +# agent: +# machine: +# type: s1-macos-15-arm64-8 +# jobs: +# - name: 'OSX arm64/m1: share consumer tests' +# commands: +# - brew install openjdk@21 +# - sudo ln -sfn $(brew --prefix openjdk@21)/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-21.jdk +# - export JAVA_HOME=$(/usr/libexec/java_home -v 21) +# - cd tests && python3 -m pip install -r requirements.txt && cd .. +# - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} +# - ./packaging/tools/run-share-consumer-tests.sh +# ${KAFKA_VERSION} ${CP_VERSION} +# - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} + + - name: 'Linux Ubuntu amd64: Share Consumer Single Broker Tests' + dependencies: [] + skip: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + prologue: + commands: + - (cd tests && python3 -m pip install -r requirements.txt) + - ./configure --install-deps --enable-werror --enable-devel + - make -j all + - make -j -C tests build + - sem-version java 17 + jobs: + - name: 'Linux Ubuntu amd64: Share Consumer Single Broker Tests' + commands: + - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} + - (cd tests && python3 -m trivup.clusters.KafkaCluster --kraft + --version ${KAFKA_VERSION} --cpversion ${CP_VERSION} + --brokers 1 + --conf '["share.coordinator.state.topic.replication.factor=1", "share.coordinator.state.topic.min.isr=1", "group.share.min.record.lock.duration.ms=1000", "transaction.state.log.replication.factor=1", "transaction.state.log.min.isr=1"]' + --cmd 'TESTS_SKIP_BEFORE=0155 python3 run-test-batches.py') + - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION} + + - name: 'Windows MSVC: share consumer tests' + dependencies: [] + skip: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-windows + env_vars: + - name: VCPKG_DISABLE_METRICS + value: 'yes' + prologue: + commands: + - cd .. + - "& .\\librdkafka\\win32\\setup-vcpkg.ps1" + - cd librdkafka + - $msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -prerelease -products * -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe); & $msbuild win32\librdkafka.sln /p:Configuration=Release /p:Platform=$env:MSVC_PLATFORM /p:PlatformToolset=v142; if ($LASTEXITCODE -ne 0) { throw "Build failed" } + - cache restore jdk21-msi-${env:CACHE_TAG} + - if (Test-Path C:\jdk21.msi) { Write-Host "JDK MSI cache HIT" } else { Write-Host "JDK MSI cache MISS - downloading"; Invoke-WebRequest -Uri "https://aka.ms/download-jdk/microsoft-jdk-21-windows-x64.msi" -OutFile C:\jdk21.msi -UseBasicParsing } + - cache store jdk21-msi-${env:CACHE_TAG} C:\jdk21.msi + - Start-Process msiexec.exe -Wait -ArgumentList '/i', 'C:\jdk21.msi', '/quiet', 'ADDLOCAL=FeatureMain,FeatureEnvironment,FeatureJarFileRunWith,FeatureJavaHome' + - $env:PATH = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("PATH", "User") + epilogue: + always: + commands: + - try { Get-Process -Name java -ErrorAction Stop | Stop-Process -Force } catch {} + jobs: + - name: 'Windows x64: share consumer tests' + env_vars: + - name: triplet + value: x64-windows + - name: MSVC_PLATFORM + value: x64 + commands: + - cache restore kafka-windows-${env:KAFKA_VERSION}-${env:CACHE_TAG} + - if (Test-Path C:\kafka.tgz) { Write-Host "Kafka tarball cache HIT" } else { Write-Host "Kafka tarball cache MISS - downloading"; Invoke-WebRequest -Uri "https://archive.apache.org/dist/kafka/${env:KAFKA_VERSION}/kafka_2.13-${env:KAFKA_VERSION}.tgz" -OutFile C:\kafka.tgz -UseBasicParsing } + - cache store kafka-windows-${env:KAFKA_VERSION}-${env:CACHE_TAG} C:\kafka.tgz + - tar -xzf C:\kafka.tgz -C C:\ + - Add-Content "C:\kafka_2.13-${env:KAFKA_VERSION}\config\server.properties" "`ngroup.share.min.record.lock.duration.ms=1000`ntransaction.state.log.replication.factor=1`ntransaction.state.log.min.isr=1" + - $env:PATH = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("PATH", "User"); $env:KAFKA_CLUSTER_ID = & "C:\kafka_2.13-${env:KAFKA_VERSION}\bin\windows\kafka-storage.bat" random-uuid; & "C:\kafka_2.13-${env:KAFKA_VERSION}\bin\windows\kafka-storage.bat" format --standalone -t $env:KAFKA_CLUSTER_ID -c "C:\kafka_2.13-${env:KAFKA_VERSION}\config\server.properties"; Start-Job -Name kafka-broker -ScriptBlock { param($v) & "C:\kafka_2.13-$v\bin\windows\kafka-server-start.bat" "C:\kafka_2.13-$v\config\server.properties" } -ArgumentList $env:KAFKA_VERSION; Start-Sleep 15; cd tests; Set-Content test.conf "bootstrap.servers=localhost:9092`nbroker.address.family=v4"; Remove-Item Env:TESTS -ErrorAction SilentlyContinue; Remove-Item Env:TESTS_SKIP -ErrorAction SilentlyContinue; $env:RDKAFKA_TEST_CONF = "test.conf"; $env:TESTS_SKIP_BEFORE = "0155"; $env:TEST_BROKER_OS = "windows"; & ..\win32\outdir\v142\x64\Release\tests.exe; Stop-Job -Name kafka-broker + - name: 'Windows x86: share consumer tests' + env_vars: + - name: triplet + value: x86-windows + - name: MSVC_PLATFORM + value: Win32 + commands: + - cache restore kafka-windows-${env:KAFKA_VERSION}-${env:CACHE_TAG} + - if (Test-Path C:\kafka.tgz) { Write-Host "Kafka tarball cache HIT" } else { Write-Host "Kafka tarball cache MISS - downloading"; Invoke-WebRequest -Uri "https://archive.apache.org/dist/kafka/${env:KAFKA_VERSION}/kafka_2.13-${env:KAFKA_VERSION}.tgz" -OutFile C:\kafka.tgz -UseBasicParsing } + - cache store kafka-windows-${env:KAFKA_VERSION}-${env:CACHE_TAG} C:\kafka.tgz + - tar -xzf C:\kafka.tgz -C C:\ + - Add-Content "C:\kafka_2.13-${env:KAFKA_VERSION}\config\server.properties" "`ngroup.share.min.record.lock.duration.ms=1000`ntransaction.state.log.replication.factor=1`ntransaction.state.log.min.isr=1" + - $env:PATH = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("PATH", "User"); $env:KAFKA_CLUSTER_ID = & "C:\kafka_2.13-${env:KAFKA_VERSION}\bin\windows\kafka-storage.bat" random-uuid; & "C:\kafka_2.13-${env:KAFKA_VERSION}\bin\windows\kafka-storage.bat" format --standalone -t $env:KAFKA_CLUSTER_ID -c "C:\kafka_2.13-${env:KAFKA_VERSION}\config\server.properties"; Start-Job -Name kafka-broker -ScriptBlock { param($v) & "C:\kafka_2.13-$v\bin\windows\kafka-server-start.bat" "C:\kafka_2.13-$v\config\server.properties" } -ArgumentList $env:KAFKA_VERSION; Start-Sleep 15; cd tests; Set-Content test.conf "bootstrap.servers=localhost:9092`nbroker.address.family=v4"; Remove-Item Env:TESTS -ErrorAction SilentlyContinue; Remove-Item Env:TESTS_SKIP -ErrorAction SilentlyContinue; $env:RDKAFKA_TEST_CONF = "test.conf"; $env:TESTS_SKIP_BEFORE = "0155"; $env:TEST_BROKER_OS = "windows"; & ..\win32\outdir\v142\Win32\Release\tests.exe; Stop-Job -Name kafka-broker + + - name: 'Linux x64: release artifact docker builds' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build: centos8 glibc +gssapi' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi + commands: + - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Build: centos8 glibc' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all + commands: + - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_x86_64:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Build: alpine musl +gssapi' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi + commands: + - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz + + - name: 'Build: alpine musl' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all + commands: + - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz + + + - name: 'Linux arm64: release artifact docker builds' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-1 + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + jobs: + - name: 'Build: centos8 glibc +gssapi' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi + commands: + - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Build: centos8 glibc' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all + commands: + - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Build: alpine musl +gssapi' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi + commands: + - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz + + - name: 'Build: alpine musl' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all + commands: + - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz + + - name: 'Linux s390x: release artifact docker builds' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + env_vars: + - name: S390X_USER + value: "ubuntu" + - name: LOCAL_KEY + value: "./s390x.prv" + prologue: + commands: + - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' + epilogue: + commands: + - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/' + - if [[ "$LOCAL_KEY" =~ ^\./.*$ ]]; then rm -f $LOCAL_KEY; fi + jobs: + - name: 'Build: centos8 glibc' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-linux__dist-centos8__arch-s390x__lnk-all + commands: + - . vault-setup + - packaging/tools/build-release-artifacts-s390x.sh --disable-gssapi quay.io/pypa/manylinux_2_28_s390x:2026.05.07-2 artifacts/librdkafka.tgz + + - name: 'Windows x64: MinGW-w64' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-windows + env_vars: + - name: CHERE_INVOKING + value: 'yes' + - name: MSYSTEM + value: UCRT64 + prologue: + commands: + # Set up msys2 + - "& .\\win32\\setup-msys2.ps1" + epilogue: + commands: + - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ } + jobs: + - name: 'Build: MinGW-w64 Dynamic' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std + commands: + - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh ./artifacts/librdkafka.tgz' + + - name: 'Build: MinGW-w64 Static' + env_vars: + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static + commands: + - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh --static ./artifacts/librdkafka.tgz' + + - name: 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019' + dependencies: [] + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-windows + env_vars: + # Disable vcpkg telemetry + - name: VCPKG_DISABLE_METRICS + value: 'yes' + prologue: + commands: + # install vcpkg in the parent directory. + - cd .. + # Setup vcpkg + - "& .\\librdkafka\\win32\\setup-vcpkg.ps1" + - cd librdkafka + epilogue: + commands: + - Get-ChildItem . -include *.dll -recurse + - Get-ChildItem . -include *.lib -recurse + - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ } + jobs: + - name: 'Build: MSVC x64' + env_vars: + - name: triplet + value: x64-windows + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std + commands: + - "& .\\win32\\msbuild.ps1 -platform x64" + - "& .\\win32\\package-zip.ps1 -platform x64" + - name: 'Build: MSVC x86' + env_vars: + - name: triplet + value: x86-windows + - name: ARTIFACT_KEY + value: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std + commands: + - "& .\\win32\\msbuild.ps1 -platform Win32" + - "& .\\win32\\package-zip.ps1 -platform Win32" + + - name: 'Packaging' + dependencies: + - 'Build documentation' + - 'OSX arm64/m1' + - 'OSX x64' + - 'Linux x64: release artifact docker builds' + - 'Linux arm64: release artifact docker builds' + - 'Linux s390x: release artifact docker builds' + - 'Windows x64: MinGW-w64' + - 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019' + run: + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-2 + jobs: + - name: 'Build NuGet and static packages' + commands: + # Get all artifacts from previous jobs in this workflow/pipeline. + - artifact pull workflow artifacts + - mkdir -p packages + # Prepare packaging tools + - cd packaging/nuget + - python3 -m pip install -U -r requirements.txt + # Create NuGet package + # We need --ignore-tag since the jobs don't add the tag to + # the artifact path, and they don't need to since these artifacts + # are part of the same workflow. + - ./release.py --directory ../../artifacts --ignore-tag --class NugetPackage ${SEMAPHORE_GIT_TAG_NAME} + - cp -v librdkafka.redist.*.nupkg ../../packages + # Create static package + - ./release.py --directory ../../artifacts --ignore-tag --class StaticPackage ${SEMAPHORE_GIT_TAG_NAME} + - cp -v librdkafka-static-bundle*.tgz ../../packages + - cd ../../ + # Copy generated docs to packages for inclusion in the tar ball + - cp -v artifacts/librdkafka-docs.tgz packages/ + # Maker super tar ball of all packages + - cd packages + - tar cvf librdkafka-packages-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tar . + # Provide some extra details + - ls -la + - sha256sum * + - cd .. + # Upload all packages to project artifact store + - artifact push project packages --destination librdkafka-packages-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID} + - echo Thank you + +promotions: + - name: Run local tests on all architectures and integration tests + pipeline_file: semaphore-integration.yml + auto_promote_on: + - result: passed + branch: + - "master" + # Manual promotion only, the auto promotion on master happens after + # integration tests pass. + - name: Run all tests + pipeline_file: run-all-tests.yml + parameters: + env_vars: + - required: true + name: TEST_KAFKA_GIT_REF + default_value: 4.0.0 + - required: true + name: TEST_TYPE + default_value: plaintext,ssl + - required: true + name: TEST_ARCHES + default_value: x86_64,aarch64 + - required: true + name: TEST_PARALLEL + default_value: "1" + # PLAIN and OAUTHBEARER and PLAIN are supported. + # into a job. Other mechanisms (SCRAM, GSSAPI) require additional + # trivup/CI fixes before they can be enabled. + - name: Run SASL PLAIN tests with share consumer + pipeline_file: run-sasl-tests.yml + parameters: + env_vars: + - required: true + name: TEST_KAFKA_GIT_REF + default_value: "4.3.0" + - name: TEST_SASL + default_value: "PLAIN" + - required: true + name: TEST_TYPE + default_value: "plaintext,ssl" + options: + - "plaintext,ssl" + - "plaintext" + - "ssl" + - required: true + name: TEST_ARCHES + default_value: "x86_64,aarch64" + options: + - "x86_64,aarch64" + - "x86_64" + - "aarch64" + - name: Run SASL OAUTHBEARER tests with share consumer + pipeline_file: run-sasl-tests.yml + parameters: + env_vars: + - required: true + name: TEST_KAFKA_GIT_REF + default_value: "4.3.0" + - name: TEST_SASL + default_value: "OAUTHBEARER" + - required: true + name: TEST_TYPE + default_value: "plaintext,ssl" + options: + - "plaintext,ssl" + - "plaintext" + - "ssl" + - required: true + name: TEST_ARCHES + default_value: "x86_64,aarch64" + options: + - "x86_64,aarch64" + - "x86_64" + - "aarch64" + - name: Run share consumer tests under ASAN+LSAN + pipeline_file: run-asan-tests.yml + - name: Run share consumer tests under TSAN + pipeline_file: run-tsan-tests.yml + - name: Run share consumer tests under valgrind + pipeline_file: run-valgrind-tests.yml diff --git a/lib/librdkafka-2.10.1/.semaphore/verify-linux-packages.yml b/lib/librdkafka-2.15.0/.semaphore/verify-linux-packages.yml similarity index 100% rename from lib/librdkafka-2.10.1/.semaphore/verify-linux-packages.yml rename to lib/librdkafka-2.15.0/.semaphore/verify-linux-packages.yml diff --git a/lib/librdkafka-2.10.1/CHANGELOG.md b/lib/librdkafka-2.15.0/CHANGELOG.md similarity index 83% rename from lib/librdkafka-2.10.1/CHANGELOG.md rename to lib/librdkafka-2.15.0/CHANGELOG.md index cb90e7d832c..e06aa0af3a0 100644 --- a/lib/librdkafka-2.10.1/CHANGELOG.md +++ b/lib/librdkafka-2.15.0/CHANGELOG.md @@ -1,3 +1,418 @@ +# librdkafka v2.15.0 + +librdkafka v2.15.0 is a feature release: + + +## [KIP-932](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka) Queues for Kafka – Now in **Preview** + +- Added a preview implementation of the **share consumer** (Queues for Kafka, + [KIP-932](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka)). + Members of a share group cooperatively consume from the same partitions with + per-record acquire/acknowledge semantics and redelivery, providing queue-like + consumption on top of Kafka. +- New `rd_kafka_share_*` public API, with a dedicated `rd_kafka_share_t` handle + created via `rd_kafka_share_consumer_new()`: + - Subscription: `rd_kafka_share_subscribe()`, `rd_kafka_share_unsubscribe()`, + `rd_kafka_share_subscription()`. + - Batch polling: `rd_kafka_share_poll()` returns an `rd_kafka_messages_t` + batch (`rd_kafka_messages_count()` / `rd_kafka_messages_get()` / + `rd_kafka_messages_destroy()`). + - Acknowledgement: `rd_kafka_share_acknowledge()`, + `rd_kafka_share_acknowledge_type()`, `rd_kafka_share_acknowledge_offset()` + with ACCEPT / RELEASE / REJECT types, and + `rd_kafka_message_delivery_count()`. + - Commit: `rd_kafka_share_commit_sync()`, `rd_kafka_share_commit_async()` and + the acknowledgement-commit callback + (`rd_kafka_share_set_acknowledgement_commit_cb()`). + - Lifecycle: `rd_kafka_share_consumer_close()`, + `rd_kafka_share_consumer_close_queue()`, `rd_kafka_share_destroy()`. +- Two acknowledgement modes selected by `share.acknowledgement.mode` + (default `implicit`; `explicit` requires the application to acknowledge every + record before the next poll). +- New `max.poll.records` property (default 500) and adjusted defaults for + several network properties for share consumers (`receive.message.max.bytes`, + `connections.max.idle.ms`, `reconnect.backoff.ms`, + `reconnect.backoff.max.ms`). +- See the *Share consumers (Queues for Kafka)* section of + [INTRODUCTION.md](INTRODUCTION.md#share-consumers-queues-for-kafka), the + *Share consumer* section in [rdkafka.h](src/rdkafka.h), and the + `examples/share_consumer*` programs. + +> [!Note] +> The [KIP-932](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka) +> share consumer is currently in **Preview** and should not be used in +> production environments. The public interfaces may change before General +> Availability, and known limitations apply (see +> [INTRODUCTION.md](INTRODUCTION.md#share-consumer-current-limitations)). The +> share consumer is single-threaded and not thread-safe by design. It requires +> a broker with share groups enabled (generally available in Apache Kafka +> 4.2.0). + +## Enhancements +* Add `aws_iam` option to `sasl.oauthbearer.metadata.authentication.type`, with a defensive stub that fails when no token-refresh callback is registered. + +## Fixes + +### General fixes + +* Issues: #5135. + Fix compilation with CMake when CURL is disabled. + The OAuthBearer OIDC code included `` under `#ifdef WITH_OAUTHBEARER_OIDC`, but + CMake always defines that macro (to 0 or 1), so CURL was required even when it was turned off. + Happening since 2.11.0 (#5136). +* Issues: #5282. + Fix `rd_atomic32_set`/`rd_atomic64_set` returning the new value instead of the previous one in CMake builds. + CMake never defined `HAVE_ATOMICS_{32,64}_ATOMIC`, so the setters used a non-atomic fallback that + returned the new value, which prevented the `ALL_BROKERS_DOWN` event from being raised under CMake. + Happening since 2.11.1 (#5136). + +### Consumer fixes + +* Issues: #5541. + Improve error handling in the KIP-848 `consumer` group protocol: + - Defer the leave heartbeat until the assignment is revoked, so a member that + exceeds `max.poll.interval.ms` rejoins cleanly instead of being rejected + with a fatal `INVALID_REQUEST`. + - Treat `GROUP_ID_NOT_FOUND` in the ConsumerGroupHeartbeat response as fatal, + except while the member is already leaving. + - Surface unexpected permanent broker-level heartbeat errors instead of + retrying them in a loop; internal transport/timeout codes keep their + existing retry/reconnect handling. + Happening since 2.12.0 (#5488) + + +# librdkafka v2.14.2 + +librdkafka v2.14.2 is a maintenance release: + +* Fix duplicate groups in `ListConsumerGroups` when multiple brokers + return the same group (#5417). +* Fix data race in timers (#5089). +* Update bundled OpenSSL, libcurl, zstd, zlib and cJSON + dependencies (#5346). + + +## Security considerations + +Bundled dependencies were upgraded as follows (see #5346): +OpenSSL 3.0.15 → 3.5.6 (LTS) for source/autoconf builds, and to 3.6.2 in +vcpkg-based packages (no LTS available in vcpkg); libcurl 8.10.1 → 8.20.0 +for source/autoconf builds and to 8.19.0 in vcpkg; zlib 1.3.1 → 1.3.2; +zstd 1.5.6 → 1.5.7; cJSON 1.7.14 → 1.7.19. + + * OpenSSL upgrade (3.0.15 → 3.5.6 LTS for source/autoconf, + 3.3.2 → 3.6.2 for vcpkg) addresses: + * CVE-2025-15467 (OpenSSL): upgraded OpenSSL to 3.5.6 (LTS) or + 3.6.2 with vcpkg as it usually doesn't provide LTS upgrades. + * Both branches (affect 3.0.15 and 3.3.2): CVE-2024-9143, + CVE-2024-13176, CVE-2025-9230, CVE-2025-68160, CVE-2025-69418, + CVE-2025-69419, CVE-2025-69420, CVE-2025-69421, CVE-2026-22795, + CVE-2026-22796, CVE-2026-28387, CVE-2026-28388, CVE-2026-28389, + CVE-2026-28390, CVE-2026-31789, CVE-2026-31790. + * Only the 3.3.x→3.6.2 vcpkg branch (3.0.15 was not affected): + CVE-2024-12797, CVE-2025-9231, CVE-2025-15468, CVE-2025-66199. + + * libcurl upgrade (8.10.1 → 8.20.0 source/autoconf, 8.10.1 → 8.19.0 + vcpkg) addresses: + * CVE-2025-14017 (libcurl): solved through upgrading to CURL 8.20.0. + LDAP module isn't present in pre-built binary, so this CVE doesn't + affect librdkafka but can still trigger automatic scanners. + * Fixed by 8.18.0 or earlier (both autoconf and vcpkg paths): + CVE-2024-9681, CVE-2024-11053, CVE-2025-0167, CVE-2025-0725, + CVE-2025-4947, CVE-2025-5025, CVE-2025-10966, CVE-2025-13034, + CVE-2025-14524, CVE-2025-14819, CVE-2025-15079, CVE-2025-15224, + CVE-2026-1965, CVE-2026-3783, CVE-2026-3784. + * Fixed only by 8.20.0 (autoconf path); vcpkg-pinned 8.19.0 still + contains these: CVE-2026-4873, CVE-2026-5545, CVE-2026-5773, + CVE-2026-6253, CVE-2026-6276, CVE-2026-6429, CVE-2026-7168. + + * zlib (1.3.1 → 1.3.2): CVE-2026-27171 (CPU exhaustion in + `crc32_combine64` and `crc32_combine_gen64`). + + * zstd (1.5.6 → 1.5.7): no CVEs; bug-fix and performance release. + + * cJSON (1.7.14 → 1.7.19): CVE-2023-50471, CVE-2023-50472, + CVE-2024-31755, CVE-2025-57052. + + +## Fixes + +### General fixes + +* Issues: #5082. + Fix data race in timers. The callback and its argument could have been modified after the lock is released. + Happening since 1.x (#5089). + +### Consumer fixes + +* Fix crash (SIGSEGV) in `rd_kafka_cgrp_handle_LeaveGroup()` when coordinator + is unavailable during consumer close. The error logging path dereferenced + a potentially NULL broker pointer. Happening since 1.x. + +### Admin client fixes + +* Issues: #5417. + Fix duplicate groups in `ListConsumerGroups` when multiple brokers return the same group. + Happening since 1.x (#5417). + + +# librdkafka v2.14.1 + +librdkafka v2.14.1 is a maintenance release: + +* Bundle prebuilt binaries for linux-s390x (#5365). + + + +# librdkafka v2.14.0 + +librdkafka v2.14.0 is a feature release: + +* [KIP-768](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=186877575#KIP768:ExtendSASL/OAUTHBEARERwithSupportforOIDC-ClientConfiguration) Extend SASL/OAUTHBEARER to support OIDC claim mapping beyond the default `sub` claim (#5336). + + + +# librdkafka v2.13.2 + +librdkafka v2.13.2 is a maintenance release: + +* The `librdkafka.redist` NuGet package now includes binary for alpine-arm64 ([#5237](https://github.com/confluentinc/librdkafka/pull/5237), [@mclayton7](https://github.com/mclayton7)) +* Remove CPU usage regression when a subscription matches no topics (#5324). +* Fix `rd_kafka_consume_batch_queue` incorrectly updating the application + position on EOF or error messages (#5213). +* Fix compilation without `getentropy` (@olegrok, @lpsinger, #5288). +* Use a truly random seed for pseudo-random number generation whenever available (#5288). +* Fix rd_list destroy callback type mismatch by changing rd_kafka_assignor_destroy to take a void * argument, as expected by rd_list_init() destroy callbacks, and casting internally to rd_kafka_assignor_t * (#5195) (#5278). + + +## Fixes + +### General fixes + +* Issues: #5283. + Fix compilation without `getentropy`. + glibc versions lacking support are those less than 2.25 (2017). + Happening since 2.13.0 (@olegrok, @lpsinger, #5288). + +### Consumer fixes + +* Issues: #5324. + Remove CPU usage regression when a subscription matches no topics. + The increased CPU usage (~30%) was seen in particular when there are many topics + in the clusters and the given subscription regex doesn't match any. + Happening since 2.10.0 (#5324). +* Issues: #4844. + Fix `rd_kafka_consume_batch_queue` incorrectly updating the application + position when receiving EOF or error messages, causing the position to + move forward and likely be stored and committed. + When storing the application offset the leader epoch is also considered for correct offset ordering in case of log truncation. + Happening since 2.2.0 (#5213). + + + +# librdkafka v2.13.0 + +librdkafka v2.13.0 is a feature release: + +* [KIP-482](https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields) Upgrade CreateAcls, DescribeAcls, DeleteAcls to the first version supporting this KIP (#5081). +* [KIP-482](https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields) Upgrade DescribeGroups, DeleteTopics, DeleteRecords, CreatePartitions, DeleteGroups to the first version supporting this KIP (#5083). +* Strip trailing dot of hostname to fix SSL certificate verification issue (#5253). +* Fix memory management for interceptors in rd_kafka_conf to prevent +double-free errors (#5240). +* Fix for the pseudo-random generator seed on Windows involving as well + the uniqueness of the new consumer group protocol member id (#5265). +* Add secure random generation functionality used for UUID uniqueness + and secure salt generation in `rd_kafka_UserScramCredentialUpsertion` + using OpenSSL or the POSIX or WIN32 equivalent calls when it + isn't available (#5265). + + +## Fixes + +### General fixes + +* Issues: #4348. + Strip trailing dot of hostname to fix SSL certificate verification issue. + Happening since 1.x (#5253). +* Issues: #4142. + Fix memory management for interceptors in rd_kafka_conf to prevent double-free errors. + In case the client instance fails the users needs to destroy the configuration + data structure, it was causing a double-free because the interceptors were + already freed in the constructor. + Happening since 1.x (#5240). +* Issues: #5263, #3929. + Fix for the pseudo-random seed on Windows. The function `rand_r` isn't present + on Windows and the global seed wasn't based on the current microseconds and thread + id. Also it wasn't called on every thread as required on this platform but + only once per process. The fix allows on this platform the uniqueness of client side + member id generation in next-generation consumer group protocol. + Happening since 1.x (#5265). + + + +# librdkafka v2.12.1 + +librdkafka v2.12.1 is a maintenance release: + +* Restored macOS binaries compatibility with macOS 13 and 14 (#5219). + + +## Fixes + +### General fixes + +* Fix to restore macOS 13 and 14 compatibility in prebuilt binaries present in `librdkafka.redist`. + Happening since 2.12.0 (#5219). + + + +# librdkafka v2.12.0 + +librdkafka v2.12.0 is a feature release: + +## [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) – General Availability + +Starting with **librdkafka 2.12.0**, the next generation consumer group rebalance protocol defined in **[KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol)** is **production-ready**. Please refer the following [migration guide](INTRODUCTION.md#next-generation-consumer-group-protocol-kip-848) for moving from `classic` to `consumer` protocol. + +**Note:** The new consumer group protocol defined in [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) is not enabled by default. There are few contract change associated with the new protocol and might cause breaking changes. `group.protocol` configuration property dictates whether to use the new `consumer` protocol or older `classic` protocol. It defaults to `classic` if not provided. + +## Enhancements and Fixes +* Support for OAUTHBEARER metadata based authentication types, + starting with Azure IMDS. [Introduction available](INTRODUCTION.md#oauthbearer-oidc-metadata-authentication) (#5155). +* Fix compression types read issue in GetTelemetrySubscriptions response + for big-endian architectures (#5183, @paravoid). +* Fix for KIP-1102 time based re-bootstrap condition (#5177). +* Fix for discarding the member epoch in a consumer group heartbeat response when leaving with an inflight HB (#4672). +* Fix for an error being raised after a commit due to an existing error in the topic partition (#4672). +* Fix double free of headers in `rd_kafka_produceva` method (@blindspotbounty, #4628). +* Fix to ensure `rd_kafka_query_watermark_offsets` enforces the specified timeout and does not continue beyond timeout expiry (#5201). +* New [walkthrough](https://github.com/confluentinc/librdkafka/wiki/Using-SASL-GSSAPI-with-librdkafka-in-a-cross%E2%80%90realm-scenario-with-Windows-SSPI-and-MIT-Kerberos) in the Wiki about configuring Kafka cross-realm authentication between Windows SSPI and MIT Kerberos. + + +## Fixes + +### General fixes + +* Issues: #5178. + Fix for KIP-1102 time based re-bootstrap condition. + Re-bootstrap is now triggered only after `metadata.recovery.rebootstrap.trigger.ms` + have passed since first metadata refresh request after last successful + metadata response. The calculation was since last successful metadata response + so it's possible it did overlap with the periodic `topic.metadata.refresh.interval.ms` + and cause a re-bootstrap even if not needed. + Happening since 2.11.0 (#5177). +* Issues: #4878. + Fix to ensure `rd_kafka_query_watermark_offsets` enforces the specified timeout and does not continue beyond timeout expiry. + Happening since 2.3.0 (#5201). + +### Telemetry fixes + +* Issues: #5179 . + Fix issue in GetTelemetrySubscriptions with big-endian + architectures where wrong values are read as + accepted compression types causing the metrics to be sent uncompressed. + Happening since 2.5.0. Since 2.10.1 unit tests are failing when run on + big-endian architectures (#5183, @paravoid). + +### Consumer fixes + + * Issues: #5199 + Fixed an issue where topic partition errors were not cleared after a successful + commit. Previously, a partition could retain a stale error state even though the + most recent commit succeeded, causing misleading error reporting. Now, successful + commits correctly clear the error state for the affected partitions + Happening since 2.4.0 (#4672). + +### Producer fixes + +* Issues: #4627. + Fix double free of headers in `rd_kafka_produceva` method in cases where the partition doesn't exist. + Happening since 1.x (@blindspotbounty, #4628). + + +# librdkafka v2.11.1 + +librdkafka v2.11.1 is a maintenance release: + +* Made the conditions for enabling the features future proof (#5130). +* Avoid returning an all brokers down error on planned disconnections (#5126). +* An "all brokers down" error isn't returned when we haven't tried to connect + to all brokers since last successful connection (#5126). + + +## Fixes + +### General fixes + +* Issues: #4948, #4956. + Made the conditions for enabling the features future proof, allowing to + remove RPC versions in a subsequent Apache Kafka version without disabling + features. The existing checks were matching a single version instead of + a range and were failing if the older version was removed. + Happening since 1.x (#5130). + +* Issues: #5142. + Avoid returning an all brokers down error on planned disconnections. + This is done by avoiding to count planned disconnections, such as idle + disconnections, broker host change and similar as events that can cause + the client to reach the "all brokers down" state, returning an error and + since 2.10.0 possibly starting a re-bootstrap sequence. + Happening since 1.x (#5126). + +* Issues: #5142. + An "all brokers down" error isn't returned when we haven't tried to connect + to all brokers since last successful connection. It happened because the down + state is cached and can be stale when a connection isn't needed to that + particular broker. Solved by resetting the cached broker down state when any + broker successfully connects, so that broker needs to be tried again. + Happening since 1.x (#5126). + + + +# librdkafka v2.11.0 + +librdkafka v2.11.0 is a feature release: + +* [KIP-1102](https://cwiki.apache.org/confluence/display/KAFKA/KIP-1102%3A+Enable+clients+to+rebootstrap+based+on+timeout+or+error+code) Enable clients to rebootstrap based on timeout or error code (#4981). +* [KIP-1139](https://cwiki.apache.org/confluence/display/KAFKA/KIP-1139%3A+Add+support+for+OAuth+jwt-bearer+grant+type) Add support for OAuth jwt-bearer grant type (#4978). +* Fix for poll ratio calculation in case the queues are forwarded (#5017). +* Fix data race when buffer queues are being reset instead of being + initialized (#4718). +* Features BROKER_BALANCED_CONSUMER and SASL_GSSAPI don't depend on + JoinGroup v0 anymore, missing in AK 4.0 and CP 8.0 (#5131). +* Improve HTTPS CA certificates configuration by probing several paths + when OpenSSL is statically linked and providing a way to customize their location + or value (#5133). + + +## Fixes + +### General fixes + +* Issues: #4522. + A data race happened when emptying buffers of a failing broker, in its thread, + with the statistics callback in main thread gathering the buffer counts. + Solved by resetting the atomic counters instead of initializing them. + Happening since 1.x (#4718). +* Issues: #4948 + Features BROKER_BALANCED_CONSUMER and SASL_GSSAPI don't depend on + JoinGroup v0 anymore, missing in AK 4.0 and CP 8.0. This PR partially + fixes the linked issue, a complete fix for all features will follow. + Rest of fixes are necessary only for a subsequent Apache Kafka major + version (e.g. AK 5.x). + Happening since 1.x (#5131). + +### Telemetry fixes + +* Issues: #5109 + Fix for poll ratio calculation in case the queues are forwarded. + Poll ratio is now calculated per-queue instead of per-instance and + it allows to avoid calculation problems linked to using the same + field. + Happens since 2.6.0 (#5017). + + + # librdkafka v2.10.1 librdkafka v2.10.1 is a maintenance release: @@ -104,8 +519,22 @@ librdkafka v2.10.0 is a feature release: > The [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) consumer is currently in **Preview** and should not be used in production environments. Implementation is feature complete but contract could have minor changes before General Availability. + ## Upgrade considerations + + + Starting from this version, brokers not reported in Metadata RPC call are + removed along with their threads. Brokers and their threads are added back + when they appear in a Metadata RPC response again. When no brokers are left + or they're not reachable, the client will start a re-bootstrap sequence + by default. `metadata.recovery.strategy` controls this, + which defaults to `rebootstrap`. + Setting `metadata.recovery.strategy` to `none` avoids any re-bootstrapping and + leaves only the broker received in last successful metadata response. + + ## Enhancements and Fixes + * [KIP-899](https://cwiki.apache.org/confluence/display/KAFKA/KIP-899%3A+Allow+producer+and+consumer+clients+to+rebootstrap) Allow producer and consumer clients to rebootstrap * Identify brokers only by broker id (#4557, @mfleming) * Remove unavailable brokers and their thread (#4557, @mfleming) * Commits during a cooperative incremental rebalance aren't causing @@ -155,7 +584,7 @@ librdkafka v2.10.0 is a feature release: and connection. Happens since 1.x (#4557, @mfleming). * Issues: #4557 - Remove brokers not reported in a metadata call, along with their thread. + Remove brokers not reported in a metadata call, along with their threads. Avoids that unavailable brokers are selected for a new connection when there's no one available. We cannot tell if a broker was removed temporarily or permanently so we always remove it and it'll be added back when diff --git a/lib/librdkafka-2.10.1/CMakeLists.txt b/lib/librdkafka-2.15.0/CMakeLists.txt similarity index 100% rename from lib/librdkafka-2.10.1/CMakeLists.txt rename to lib/librdkafka-2.15.0/CMakeLists.txt diff --git a/lib/librdkafka-2.10.1/CODE_OF_CONDUCT.md b/lib/librdkafka-2.15.0/CODE_OF_CONDUCT.md similarity index 100% rename from lib/librdkafka-2.10.1/CODE_OF_CONDUCT.md rename to lib/librdkafka-2.15.0/CODE_OF_CONDUCT.md diff --git a/lib/librdkafka-2.10.1/CONFIGURATION.md b/lib/librdkafka-2.15.0/CONFIGURATION.md similarity index 84% rename from lib/librdkafka-2.10.1/CONFIGURATION.md rename to lib/librdkafka-2.15.0/CONFIGURATION.md index b32f3e6d115..974ac154bc5 100644 --- a/lib/librdkafka-2.10.1/CONFIGURATION.md +++ b/lib/librdkafka-2.15.0/CONFIGURATION.md @@ -8,18 +8,19 @@ client.id | * | | rdkafka metadata.broker.list | * | | | high | Initial list of brokers as a CSV list of broker host or host:port. The application may also use `rd_kafka_brokers_add()` to add brokers during runtime.
*Type: string* bootstrap.servers | * | | | high | Alias for `metadata.broker.list`: Initial list of brokers as a CSV list of broker host or host:port. The application may also use `rd_kafka_brokers_add()` to add brokers during runtime.
*Type: string* message.max.bytes | * | 1000 .. 1000000000 | 1000000 | medium | Maximum Kafka protocol request message size. Due to differing framing overhead between protocol versions the producer is unable to reliably enforce a strict max message limit at produce time and may exceed the maximum size by one message in protocol ProduceRequests, the broker will enforce the the topic's `max.message.bytes` limit (see Apache Kafka documentation).
*Type: integer* -message.copy.max.bytes | * | 0 .. 1000000000 | 65535 | low | Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs.
*Type: integer* -receive.message.max.bytes | * | 1000 .. 2147483647 | 100000000 | medium | Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value must be at least `fetch.max.bytes` + 512 to allow for protocol overhead; the value is adjusted automatically unless the configuration property is explicitly set.
*Type: integer* -max.in.flight.requests.per.connection | * | 1 .. 1000000 | 1000000 | low | Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one.
*Type: integer* -max.in.flight | * | 1 .. 1000000 | 1000000 | low | Alias for `max.in.flight.requests.per.connection`: Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one.
*Type: integer* -metadata.recovery.strategy | * | none, rebootstrap | rebootstrap | low | Controls how the client recovers when none of the brokers known to it is available. If set to `none`, the client fails with a fatal error. If set to `rebootstrap`, the client repeats the bootstrap process using `bootstrap.servers` and brokers added through `rd_kafka_brokers_add()`. Rebootstrapping is useful when a client communicates with brokers so infrequently that the set of brokers may change entirely before the client refreshes metadata. Metadata recovery is triggered when all last-known brokers appear unavailable simultaneously.
*Type: enum value* +message.copy.max.bytes | * | 0 .. 1000000000 | 65535 | low | Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs. This property is not supported for share consumers.
*Type: integer* +receive.message.max.bytes | * | 1000 .. 2147483647 | 100000000 | medium | Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value must be at least `fetch.max.bytes` + 512 to allow for protocol overhead; the value is adjusted automatically unless the configuration property is explicitly set. For share consumers, the default value is INT_MAX.
*Type: integer* +max.in.flight.requests.per.connection | * | 1 .. 1000000 | 1000000 | low | Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. This property is ignored for share consumers.
*Type: integer* +max.in.flight | * | 1 .. 1000000 | 1000000 | low | Alias for `max.in.flight.requests.per.connection`: Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. This property is ignored for share consumers.
*Type: integer* +metadata.recovery.strategy | * | none, rebootstrap | rebootstrap | low | Controls how the client recovers when none of the brokers known to it is available. If set to `none`, the client doesn't re-bootstrap. If set to `rebootstrap`, the client repeats the bootstrap process using `bootstrap.servers` and brokers added through `rd_kafka_brokers_add()`. Rebootstrapping is useful when a client communicates with brokers so infrequently that the set of brokers may change entirely before the client refreshes metadata. Metadata recovery is triggered when all last-known brokers appear unavailable simultaneously or the client cannot refresh metadata within `metadata.recovery.rebootstrap.trigger.ms` or it's requested in a metadata response.
*Type: enum value* +metadata.recovery.rebootstrap.trigger.ms | * | 0 .. 2147483647 | 300000 | low | If a client configured to rebootstrap using `metadata.recovery.strategy=rebootstrap` is unable to obtain metadata from any of the brokers for this interval, client repeats the bootstrap process using `bootstrap.servers` configuration and brokers added through `rd_kafka_brokers_add()`.
*Type: integer* topic.metadata.refresh.interval.ms | * | -1 .. 3600000 | 300000 | low | Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s.
*Type: integer* metadata.max.age.ms | * | 1 .. 86400000 | 900000 | low | Metadata cache max age. Defaults to topic.metadata.refresh.interval.ms * 3
*Type: integer* topic.metadata.refresh.fast.interval.ms | * | 1 .. 60000 | 100 | low | When a topic loses its leader a new metadata request will be enqueued immediately and then with this initial interval, exponentially increasing upto `retry.backoff.max.ms`, until the topic metadata has been refreshed. If not set explicitly, it will be defaulted to `retry.backoff.ms`. This is used to recover quickly from transitioning leader brokers.
*Type: integer* topic.metadata.refresh.fast.cnt | * | 0 .. 1000 | 10 | low | **DEPRECATED** No longer used.
*Type: integer* topic.metadata.refresh.sparse | * | true, false | true | low | Sparse metadata requests (consumes less network bandwidth)
*Type: boolean* topic.metadata.propagation.max.ms | * | 0 .. 3600000 | 30000 | low | Apache Kafka topic creation is asynchronous and it takes some time for a new topic to propagate throughout the cluster to all brokers. If a client requests topic metadata after manual topic creation but before the topic has been fully propagated to the broker the client is requesting metadata from, the topic will seem to be non-existent and the client will mark the topic as such, failing queued produced messages with `ERR__UNKNOWN_TOPIC`. This setting delays marking a topic as non-existent until the configured propagation max time has passed. The maximum propagation time is calculated from the time the topic is first referenced in the client, e.g., on produce().
*Type: integer* -topic.blacklist | * | | | low | Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist.
*Type: pattern list* +topic.blacklist | * | | | low | Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist. This property is not supported for share consumers.
*Type: pattern list* debug | * | generic, broker, topic, metadata, feature, queue, msg, protocol, cgrp, security, fetch, interceptor, plugin, consumer, admin, eos, mock, assignor, conf, telemetry, all | | medium | A comma-separated list of debug contexts to enable. Detailed Producer debugging: broker,topic,msg. Consumer: consumer,cgrp,topic,fetch
*Type: CSV flags* socket.timeout.ms | * | 10 .. 300000 | 60000 | low | Default timeout for network requests. Producer: ProduceRequests will use the lesser value of `socket.timeout.ms` and remaining `message.timeout.ms` for the first message in the batch. Consumer: FetchRequests will use `fetch.wait.max.ms` + `socket.timeout.ms`. Admin: Admin requests will use `socket.timeout.ms` or explicitly set `rd_kafka_AdminOptions_set_operation_timeout()` value.
*Type: integer* socket.blocking.max.ms | * | 1 .. 60000 | 1000 | low | **DEPRECATED** No longer used.
*Type: integer* @@ -31,10 +32,10 @@ socket.max.fails | * | 0 .. 1000000 | 1 broker.address.ttl | * | 0 .. 86400000 | 1000 | low | How long to cache the broker address resolving results (milliseconds).
*Type: integer* broker.address.family | * | any, v4, v6 | any | low | Allowed broker IP address families: any, v4, v6
*Type: enum value* socket.connection.setup.timeout.ms | * | 1000 .. 2147483647 | 30000 | medium | Maximum time allowed for broker connection setup (TCP connection setup as well SSL and SASL handshake). If the connection to the broker is not fully functional after this the connection will be closed and retried.
*Type: integer* -connections.max.idle.ms | * | 0 .. 2147483647 | 0 | medium | Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info).
*Type: integer* +connections.max.idle.ms | * | 0 .. 2147483647 | 0 | medium | Close broker connections after the specified time of inactivity. Disable with 0. For share consumers, the default value is 540000 (9 mins).If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info). Actual value can be lower, up to 2s lower, only if `connections.max.idle.ms` >= 4s, as jitter is added to avoid disconnecting all brokers at the same time.
*Type: integer* reconnect.backoff.jitter.ms | * | 0 .. 3600000 | 0 | low | **DEPRECATED** No longer used. See `reconnect.backoff.ms` and `reconnect.backoff.max.ms`.
*Type: integer* -reconnect.backoff.ms | * | 0 .. 3600000 | 100 | medium | The initial time to wait before reconnecting to a broker after the connection has been closed. The time is increased exponentially until `reconnect.backoff.max.ms` is reached. -25% to +50% jitter is applied to each reconnect backoff. A value of 0 disables the backoff and reconnects immediately.
*Type: integer* -reconnect.backoff.max.ms | * | 0 .. 3600000 | 10000 | medium | The maximum time to wait before reconnecting to a broker after the connection has been closed.
*Type: integer* +reconnect.backoff.ms | * | 0 .. 3600000 | 100 | medium | The initial time to wait before reconnecting to a broker after the connection has been closed. The time is increased exponentially until `reconnect.backoff.max.ms` is reached. -25% to +50% jitter is applied to each reconnect backoff. A value of 0 disables the backoff and reconnects immediately. For share consumers, the default value is 50.
*Type: integer* +reconnect.backoff.max.ms | * | 0 .. 3600000 | 10000 | medium | The maximum time to wait before reconnecting to a broker after the connection has been closed. For share consumers, the default value is 1000.
*Type: integer* statistics.interval.ms | * | 0 .. 86400000 | 0 | high | librdkafka statistics emit interval. The application also needs to register a stats callback using `rd_kafka_conf_set_stats_cb()`. The granularity is 1000ms. A value of 0 disables statistics.
*Type: integer* enabled_events | * | 0 .. 2147483647 | 0 | low | See `rd_kafka_conf_set_events()`
*Type: integer* error_cb | * | | | low | Error callback (set with rd_kafka_conf_set_error_cb())
*Type: see dedicated API* @@ -59,7 +60,7 @@ api.version.request | * | true, false | true api.version.request.timeout.ms | * | 1 .. 300000 | 10000 | low | Timeout for broker API version requests.
*Type: integer* api.version.fallback.ms | * | 0 .. 604800000 | 0 | medium | **DEPRECATED** **Post-deprecation actions: remove this configuration property, brokers < 0.10.0 won't be supported anymore in librdkafka 3.x.** Dictates how long the `broker.version.fallback` fallback is used in the case the ApiVersionRequest fails. **NOTE**: The ApiVersionRequest is only issued when a new connection to the broker is made (such as after an upgrade).
*Type: integer* broker.version.fallback | * | | 0.10.0 | medium | **DEPRECATED** **Post-deprecation actions: remove this configuration property, brokers < 0.10.0 won't be supported anymore in librdkafka 3.x.** Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests.
*Type: string* -allow.auto.create.topics | * | true, false | false | low | Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. The broker must also be configured with `auto.create.topics.enable=true` for this configuration to take effect. Note: the default value (true) for the producer is different from the default value (false) for the consumer. Further, the consumer default value is different from the Java consumer (true), and this property is not supported by the Java producer. Requires broker version >= 0.11.0.0, for older broker versions only the broker configuration applies.
*Type: boolean* +allow.auto.create.topics | * | true, false | false | low | Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. The broker must also be configured with `auto.create.topics.enable=true` for this configuration to take effect. Note: the default value (true) for the producer is different from the default value (false) for the consumer. Further, the consumer default value is different from the Java consumer (true), and this property is not supported by the Java producer. Requires broker version >= 0.11.0.0, for older broker versions only the broker configuration applies. This property is currently not supported for share consumers and will be enabled in the General Availability (GA) release.
*Type: boolean* security.protocol | * | plaintext, ssl, sasl_plaintext, sasl_ssl | plaintext | high | Protocol used to communicate with brokers.
*Type: enum value* ssl.cipher.suites | * | | | low | A cipher suite is a named combination of authentication, encryption, MAC and key exchange algorithm used to negotiate the security settings for a network connection using TLS or SSL network protocol. See manual page for `ciphers(1)` and `SSL_CTX_set_cipher_list(3).
*Type: string* ssl.curves.list | * | | | low | The supported-curves extension in the TLS ClientHello message specifies the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client is willing to have the server use. See manual page for `SSL_CTX_set1_curves_list(3)`. OpenSSL >= 1.0.2 required.
*Type: string* @@ -72,6 +73,8 @@ ssl.certificate.location | * | | ssl.certificate.pem | * | | | low | Client's public key string (PEM format) used for authentication.
*Type: string* ssl_certificate | * | | | low | Client's public key as set by rd_kafka_conf_set_ssl_cert()
*Type: see dedicated API* ssl.ca.location | * | | | low | File or directory path to CA certificate(s) for verifying the broker's key. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to `probe`. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or `ssl.ca.location` is set to `probe` a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see `OPENSSLDIR` in `openssl version -a`).
*Type: string* +https.ca.location | * | | | low | File or directory path to CA certificate(s) for verifying HTTPS endpoints, like `sasl.oauthbearer.token.endpoint.url` used for OAUTHBEARER/OIDC authentication. Mutually exclusive with `https.ca.pem`. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to `probe`. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or `https.ca.location` is set to `probe` a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see `OPENSSLDIR` in `openssl version -a`).
*Type: string* +https.ca.pem | * | | | low | CA certificate string (PEM format) for verifying HTTPS endpoints. Mutually exclusive with `https.ca.location`. Optional: see `https.ca.location`.
*Type: string* ssl.ca.pem | * | | | low | CA certificate string (PEM format) for verifying the broker's key.
*Type: string* ssl_ca | * | | | low | CA certificate as set by rd_kafka_conf_set_ssl_cert()
*Type: see dedicated API* ssl.ca.certificate.stores | * | | Root | low | Comma-separated list of Windows Certificate stores to load CA certificates from. Certificates will be loaded in the same order as stores are specified. If no certificates can be loaded from any of the specified stores an error is logged and the OpenSSL library's default CA location is used instead. Store names are typically one or more of: MY, Root, Trust, CA.
*Type: string* @@ -99,42 +102,61 @@ enable.sasl.oauthbearer.unsecure.jwt | * | true, false | false oauthbearer_token_refresh_cb | * | | | low | SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token. Also see `rd_kafka_conf_enable_sasl_queue()`.
*Type: see dedicated API* sasl.oauthbearer.method | * | default, oidc | default | low | Set to "default" or "oidc" to control which login method to be used. If set to "oidc", the following properties must also be be specified: `sasl.oauthbearer.client.id`, `sasl.oauthbearer.client.secret`, and `sasl.oauthbearer.token.endpoint.url`.
*Type: enum value* sasl.oauthbearer.client.id | * | | | low | Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* +sasl.oauthbearer.client.credentials.client.id | * | | | low | Alias for `sasl.oauthbearer.client.id`: Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* +sasl.oauthbearer.client.credentials.client.secret | * | | | low | Alias for `sasl.oauthbearer.client.secret`: Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* sasl.oauthbearer.client.secret | * | | | low | Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* sasl.oauthbearer.scope | * | | | low | Client use this to specify the scope of the access request to the broker. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* sasl.oauthbearer.extensions | * | | | low | Allow additional information to be provided to the broker. Comma-separated list of key=value pairs. E.g., "supportFeatureX=true,organizationId=sales-emea".Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* sasl.oauthbearer.token.endpoint.url | * | | | low | OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* +sasl.oauthbearer.sub.claim.name | * | | sub | low | JWT claim name to use as the subject (principal) when validating OIDC access tokens. Must be present in the JWT payload with a non-empty value. Should match the broker's `sasl.oauthbearer.sub.claim.name` configuration for consistent authentication. Only used when `sasl.oauthbearer.method` is set to "oidc".
*Type: string* +sasl.oauthbearer.grant.type | * | client_credentials, urn:ietf:params:oauth:grant-type:jwt-bearer | client_credentials | low | OAuth grant type to use when communicating with the identity provider.
*Type: enum value* +sasl.oauthbearer.assertion.algorithm | * | RS256, ES256 | RS256 | low | Algorithm the client should use to sign the assertion sent to the identity provider and in the OAuth alg header in the JWT assertion.
*Type: enum value* +sasl.oauthbearer.assertion.private.key.file | * | | | low | Path to client's private key (PEM) used for authentication when using the JWT assertion.
*Type: string* +sasl.oauthbearer.assertion.private.key.passphrase | * | | | low | Private key passphrase for `sasl.oauthbearer.assertion.private.key.file` or `sasl.oauthbearer.assertion.private.key.pem`.
*Type: string* +sasl.oauthbearer.assertion.private.key.pem | * | | | low | Client's private key (PEM) used for authentication when using the JWT assertion.
*Type: string* +sasl.oauthbearer.assertion.file | * | | | low | Path to the assertion file. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: string* +sasl.oauthbearer.assertion.claim.aud | * | | | low | JWT audience claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: string* +sasl.oauthbearer.assertion.claim.exp.seconds | * | 1 .. 2147483647 | 300 | low | Assertion expiration time in seconds. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: integer* +sasl.oauthbearer.assertion.claim.iss | * | | | low | JWT issuer claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: string* +sasl.oauthbearer.assertion.claim.jti.include | * | true, false | false | low | JWT ID claim. When set to `true`, a random UUID is generated. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: boolean* +sasl.oauthbearer.assertion.claim.nbf.seconds | * | 0 .. 2147483647 | 60 | low | Assertion not before time in seconds. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: integer* +sasl.oauthbearer.assertion.claim.sub | * | | | low | JWT subject claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: string* +sasl.oauthbearer.assertion.jwt.template.file | * | | | low | Path to the JWT template file. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
*Type: string* +sasl.oauthbearer.metadata.authentication.type | * | none, azure_imds, aws_iam | none | low | Type of metadata-based authentication to use for OAUTHBEARER/OIDC `azure_imds` authenticates using the Azure IMDS endpoint. Sets a default value for `sasl.oauthbearer.token.endpoint.url` if missing. `aws_iam` indicates AWS IAM-based authentication using GetWebIdentityToken API. librdkafka does not implement the aws_iam token flow natively as of now. Configuration values specific of chosen authentication type can be passed through `sasl.oauthbearer.config`.
*Type: enum value* plugin.library.paths | * | | | low | List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically.
*Type: string* interceptors | * | | | low | Interceptors added through rd_kafka_conf_interceptor_add_..() and any configuration handled by interceptors.
*Type: see dedicated API* group.id | C | | | high | Client group id string. All clients sharing the same group.id belong to the same group.
*Type: string* -group.instance.id | C | | | medium | Enable static group membership. Static group members are able to leave and rejoin a group within the configured `session.timeout.ms` without prompting a group rebalance. This should be used in combination with a larger `session.timeout.ms` to avoid group rebalances caused by transient unavailability (e.g. process restarts). Requires broker version >= 2.3.0.
*Type: string* +group.instance.id | C | | | medium | Enable static group membership. Static group members are able to leave and rejoin a group within the configured `session.timeout.ms` without prompting a group rebalance. This should be used in combination with a larger `session.timeout.ms` to avoid group rebalances caused by transient unavailability (e.g. process restarts). Requires broker version >= 2.3.0. This property is not supported for share consumers.
*Type: string* partition.assignment.strategy | C | | range,roundrobin | medium | The name of one or more partition assignment strategies. The elected group leader will use a strategy supported by all members of the group to assign partitions to group members. If there is more than one eligible strategy, preference is determined by the order of this list (strategies earlier in the list have higher priority). Cooperative and non-cooperative (eager)strategies must not be mixed. `partition.assignment.strategy` is not supported for `group.protocol=consumer`. Use `group.remote.assignor` instead. Available strategies: range, roundrobin, cooperative-sticky.
*Type: string* session.timeout.ms | C | 1 .. 3600000 | 45000 | high | Client group session and failure detection timeout. The consumer sends periodic heartbeats (heartbeat.interval.ms) to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is configured with the **broker** configuration properties `group.min.session.timeout.ms` and `group.max.session.timeout.ms`. `session.timeout.ms` is not supported for `group.protocol=consumer`. It is set with the broker configuration property `group.consumer.session.timeout.ms` by default or can be configured through the AdminClient IncrementalAlterConfigs API. The allowed range is configured with the broker configuration properties `group.consumer.min.session.timeout.ms` and `group.consumer.max.session.timeout.ms`. Also see `max.poll.interval.ms`.
*Type: integer* heartbeat.interval.ms | C | 1 .. 3600000 | 3000 | low | Group session keepalive heartbeat interval. `heartbeat.interval.ms` is not supported for `group.protocol=consumer`. It is set with the broker configuration property `group.consumer.heartbeat.interval.ms` by default or can be configured through the AdminClient IncrementalAlterConfigs API. The allowed range is configured with the broker configuration properties `group.consumer.min.heartbeat.interval.ms` and `group.consumer.max.heartbeat.interval.ms`.
*Type: integer* group.protocol.type | C | | consumer | low | Group protocol type for the `classic` group protocol. NOTE: Currently, the only supported group protocol type is `consumer`. `group.protocol.type` is not supported for `group.protocol=consumer`
*Type: string* group.protocol | C | classic, consumer | classic | high | Group protocol to use. Use `classic` for the original protocol and `consumer` for the new protocol introduced in KIP-848. Available protocols: classic or consumer. Default is `classic`, but will change to `consumer` in next releases.
*Type: enum value* -group.remote.assignor | C | | | medium | Server side assignor to use. Keep it null to make server select a suitable assignor for the group. Available assignors: uniform or range. Default is null
*Type: string* +group.remote.assignor | C | | | medium | Server side assignor to use. Keep it null to make server select a suitable assignor for the group. Available assignors: uniform or range. Default is null. This property is not supported for share consumers.
*Type: string* coordinator.query.interval.ms | C | 1 .. 3600000 | 600000 | low | How often to query for the current client group coordinator. If the currently assigned coordinator is down the configured query interval will be divided by ten to more quickly recover in case of coordinator reassignment.
*Type: integer* max.poll.interval.ms | C | 1 .. 86400000 | 300000 | high | Maximum allowed time between calls to consume messages (e.g., rd_kafka_consumer_poll()) for high-level consumers. If this interval is exceeded the consumer is considered failed and the group will rebalance in order to reassign the partitions to another consumer group member. Warning: Offset commits may be not possible at this point. Note: It is recommended to set `enable.auto.offset.store=false` for long-time processing applications and then explicitly store offsets (using offsets_store()) *after* message processing, to make sure offsets are not auto-committed prior to processing has finished. The interval is checked two times per second. See KIP-62 for more information.
*Type: integer* enable.auto.commit | C | true, false | true | high | Automatically and periodically commit offsets in the background. Note: setting this to false does not prevent the consumer from fetching previously committed start offsets. To circumvent this behaviour set specific start offsets per partition in the call to assign().
*Type: boolean* -auto.commit.interval.ms | C | 0 .. 86400000 | 5000 | medium | The frequency in milliseconds that the consumer offsets are committed (written) to offset storage. (0 = disable). This setting is used by the high-level consumer.
*Type: integer* +auto.commit.interval.ms | C | 0 .. 86400000 | 5000 | medium | The frequency in milliseconds that the consumer offsets are committed (written) to offset storage. (0 = disable). This setting is used by the high-level consumer. This property is ignored for share consumers.
*Type: integer* enable.auto.offset.store | C | true, false | true | high | Automatically store offset of last message provided to application. The offset store is an in-memory store of the next offset to (auto-)commit for each partition.
*Type: boolean* -queued.min.messages | C | 1 .. 10000000 | 100000 | medium | Minimum number of messages per topic+partition librdkafka tries to maintain in the local consumer queue.
*Type: integer* -queued.max.messages.kbytes | C | 1 .. 2097151 | 65536 | medium | Maximum number of kilobytes of queued pre-fetched messages in the local consumer queue. If using the high-level consumer this setting applies to the single consumer queue, regardless of the number of partitions. When using the legacy simple consumer or when separate partition queues are used this setting applies per partition. This value may be overshot by fetch.message.max.bytes. This property has higher priority than queued.min.messages.
*Type: integer* +queued.min.messages | C | 1 .. 10000000 | 100000 | medium | Minimum number of messages per topic+partition librdkafka tries to maintain in the local consumer queue. This property is not supported for share consumers.
*Type: integer* +queued.max.messages.kbytes | C | 1 .. 2097151 | 65536 | medium | Maximum number of kilobytes of queued pre-fetched messages in the local consumer queue. If using the high-level consumer this setting applies to the single consumer queue, regardless of the number of partitions. When using the legacy simple consumer or when separate partition queues are used this setting applies per partition. This value may be overshot by fetch.message.max.bytes. This property has higher priority than queued.min.messages. This property is not supported for share consumers.
*Type: integer* fetch.wait.max.ms | C | 0 .. 300000 | 500 | low | Maximum time the broker may wait to fill the Fetch response with fetch.min.bytes of messages.
*Type: integer* -fetch.queue.backoff.ms | C | 0 .. 300000 | 1000 | medium | How long to postpone the next fetch request for a topic+partition in case the current fetch queue thresholds (queued.min.messages or queued.max.messages.kbytes) have been exceded. This property may need to be decreased if the queue thresholds are set low and the application is experiencing long (~1s) delays between messages. Low values may increase CPU utilization.
*Type: integer* -fetch.message.max.bytes | C | 1 .. 1000000000 | 1048576 | medium | Initial maximum number of bytes per topic+partition to request when fetching messages from the broker. If the client encounters a message larger than this value it will gradually try to increase it until the entire message can be fetched.
*Type: integer* -max.partition.fetch.bytes | C | 1 .. 1000000000 | 1048576 | medium | Alias for `fetch.message.max.bytes`: Initial maximum number of bytes per topic+partition to request when fetching messages from the broker. If the client encounters a message larger than this value it will gradually try to increase it until the entire message can be fetched.
*Type: integer* -fetch.max.bytes | C | 0 .. 2147483135 | 52428800 | medium | Maximum amount of data the broker shall return for a Fetch request. Messages are fetched in batches by the consumer and if the first message batch in the first non-empty partition of the Fetch request is larger than this value, then the message batch will still be returned to ensure the consumer can make progress. The maximum message batch size accepted by the broker is defined via `message.max.bytes` (broker config) or `max.message.bytes` (broker topic config). `fetch.max.bytes` is automatically adjusted upwards to be at least `message.max.bytes` (consumer config).
*Type: integer* -fetch.min.bytes | C | 1 .. 100000000 | 1 | low | Minimum number of bytes the broker responds with. If fetch.wait.max.ms expires the accumulated data will be sent to the client regardless of this setting.
*Type: integer* -fetch.error.backoff.ms | C | 0 .. 300000 | 500 | medium | How long to postpone the next fetch request for a topic+partition in case of a fetch error.
*Type: integer* +fetch.queue.backoff.ms | C | 0 .. 300000 | 1000 | medium | How long to postpone the next fetch request for a topic+partition in case the current fetch queue thresholds (queued.min.messages or queued.max.messages.kbytes) have been exceded. This property may need to be decreased if the queue thresholds are set low and the application is experiencing long (~1s) delays between messages. Low values may increase CPU utilization. This property is not supported for share consumers.
*Type: integer* +fetch.message.max.bytes | C | 1 .. 1000000000 | 1048576 | medium | Initial maximum number of bytes per topic+partition to request when fetching messages from the broker. If the client encounters a message larger than this value it will gradually try to increase it until the entire message can be fetched. This property is ignored for share consumers.
*Type: integer* +max.partition.fetch.bytes | C | 1 .. 1000000000 | 1048576 | medium | Alias for `fetch.message.max.bytes`: Initial maximum number of bytes per topic+partition to request when fetching messages from the broker. If the client encounters a message larger than this value it will gradually try to increase it until the entire message can be fetched. This property is ignored for share consumers.
*Type: integer* +fetch.max.bytes | C | 0 .. 2147483135 | 52428800 | medium | Maximum amount of data the broker shall return for a Fetch request. Messages are fetched in batches by the consumer and if the first message batch in the first non-empty partition of the Fetch request is larger than this value, then the message batch will still be returned to ensure the consumer can make progress. The maximum message batch size accepted by the broker is defined via `message.max.bytes` (broker config) or `max.message.bytes` (broker topic config). For regular (not share) consumers, `fetch.max.bytes` is automatically adjusted upwards to be at least `message.max.bytes` (consumer config).
*Type: integer* +fetch.min.bytes | C | 0 .. 2147483647 | 1 | low | Minimum number of bytes the broker responds with. If fetch.wait.max.ms expires the accumulated data will be sent to the client regardless of this setting. For regular consumers, this value must be in range 1..100000000
*Type: integer* +fetch.error.backoff.ms | C | 0 .. 300000 | 500 | medium | How long to postpone the next fetch request for a topic+partition in case of a fetch error. This property is not supported for share consumers.
*Type: integer* offset.store.method | C | none, file, broker | broker | low | **DEPRECATED** Offset commit store method: 'file' - DEPRECATED: local file store (offset.store.path, et.al), 'broker' - broker commit store (requires Apache Kafka 0.8.2 or later on the broker).
*Type: enum value* -isolation.level | C | read_uncommitted, read_committed | read_committed | high | Controls how to read messages written transactionally: `read_committed` - only return transactional messages which have been committed. `read_uncommitted` - return all messages, even transactional messages which have been aborted.
*Type: enum value* -consume_cb | C | | | low | Message consume callback (set with rd_kafka_conf_set_consume_cb())
*Type: see dedicated API* -rebalance_cb | C | | | low | Called after consumer group has been rebalanced (set with rd_kafka_conf_set_rebalance_cb())
*Type: see dedicated API* -offset_commit_cb | C | | | low | Offset commit result propagation callback. (set with rd_kafka_conf_set_offset_commit_cb())
*Type: see dedicated API* -enable.partition.eof | C | true, false | false | low | Emit RD_KAFKA_RESP_ERR__PARTITION_EOF event whenever the consumer reaches the end of a partition.
*Type: boolean* +isolation.level | C | read_uncommitted, read_committed | read_committed | high | Controls how to read messages written transactionally: `read_committed` - only return transactional messages which have been committed. `read_uncommitted` - return all messages, even transactional messages which have been aborted. This property is not supported for share consumers.
*Type: enum value* +consume_cb | C | | | low | Message consume callback (set with rd_kafka_conf_set_consume_cb()). This property is not supported for share consumers.
*Type: see dedicated API* +rebalance_cb | C | | | low | Called after consumer group has been rebalanced (set with rd_kafka_conf_set_rebalance_cb()). This property is not supported for share consumers.
*Type: see dedicated API* +offset_commit_cb | C | | | low | Offset commit result propagation callback. (set with rd_kafka_conf_set_offset_commit_cb()). This property is not supported for share consumers.
*Type: see dedicated API* +enable.partition.eof | C | true, false | false | low | Emit RD_KAFKA_RESP_ERR__PARTITION_EOF event whenever the consumer reaches the end of a partition. This property is not supported for share consumers.
*Type: boolean* check.crcs | C | true, false | false | medium | Verify CRC32 of consumed messages, ensuring no on-the-wire or on-disk corruption to the messages occurred. This check comes at slightly increased CPU usage.
*Type: boolean* client.rack | * | | | low | A rack identifier for this client. This can be any string value which indicates where this client is physically located. It corresponds with the broker config `broker.rack`.
*Type: string* +max.poll.records | C | 1 .. 2147483647 | 500 | low | The maximum number of records returned in a single call to `rd_kafka_share_poll()`. This value is sent to the broker in the ShareFetch request and therefore bounds the number of records the broker acquires and returns per fetch. Note: this limit is currently best-effort and not strictly enforced, so a poll may occasionally return more records than this value. This property is only supported for share consumers.
*Type: integer* +share.acknowledgement.mode | C | | implicit | medium | Acknowledgement mode for share consumers. 'implicit' - messages are implicitly acknowledged when the next poll is called. 'explicit' - messages must be explicitly acknowledged using rd_kafka_share_acknowledge*() APIs.
*Type: string* transactional.id | P | | | high | Enables the transactional producer. The transactional.id is used to identify the same transactional producer instance across process restarts. It allows the producer to guarantee that transactions corresponding to earlier instances of the same producer have been finalized prior to starting any new transactions, and that any zombie instances are fenced off. If no transactional.id is provided, then the producer is limited to idempotent delivery (if enable.idempotence is set). Requires broker version >= 0.11.0.
*Type: string* transaction.timeout.ms | P | 1000 .. 2147483647 | 60000 | medium | The maximum amount of time in milliseconds that the transaction coordinator will wait for a transaction status update from the producer before proactively aborting the ongoing transaction. If this value is larger than the `transaction.max.timeout.ms` setting in the broker, the init_transactions() call will fail with ERR_INVALID_TRANSACTION_TIMEOUT. The transaction timeout automatically adjusts `message.timeout.ms` and `socket.timeout.ms`, unless explicitly configured in which case they must not exceed the transaction timeout (`socket.timeout.ms` must be at least 100ms lower than `transaction.timeout.ms`). This is also the default timeout value if no timeout (-1) is supplied to the transactional API methods.
*Type: integer* enable.idempotence | P | true, false | false | high | When set to `true`, the producer will ensure that messages are successfully produced exactly once and in the original produce order. The following configuration properties are adjusted automatically (if not modified by the user) when idempotence is enabled: `max.in.flight.requests.per.connection=5` (must be less than or equal to 5), `retries=INT32_MAX` (must be greater than 0), `acks=all`, `queuing.strategy=fifo`. Producer instantation will fail if user-supplied configuration is incompatible.
*Type: boolean* @@ -180,11 +202,11 @@ compression.type | P | none, gzip, snappy, lz4, zstd | compression.level | P | -1 .. 12 | -1 | medium | Compression level parameter for algorithm selected by configuration property `compression.codec`. Higher values will result in better compression at the cost of more CPU usage. Usable range is algorithm-dependent: [0-9] for gzip; [0-12] for lz4; only 0 for snappy; -1 = codec-dependent default compression level.
*Type: integer* auto.commit.enable | C | true, false | true | low | **DEPRECATED** [**LEGACY PROPERTY:** This property is used by the simple legacy consumer only. When using the high-level KafkaConsumer, the global `enable.auto.commit` property must be used instead]. If true, periodically commit offset of the last message handed to the application. This committed offset will be used when the process restarts to pick up where it left off. If false, the application will have to call `rd_kafka_offset_store()` to store an offset (optional). Offsets will be written to broker or local file according to offset.store.method.
*Type: boolean* enable.auto.commit | C | true, false | true | low | **DEPRECATED** Alias for `auto.commit.enable`: [**LEGACY PROPERTY:** This property is used by the simple legacy consumer only. When using the high-level KafkaConsumer, the global `enable.auto.commit` property must be used instead]. If true, periodically commit offset of the last message handed to the application. This committed offset will be used when the process restarts to pick up where it left off. If false, the application will have to call `rd_kafka_offset_store()` to store an offset (optional). Offsets will be written to broker or local file according to offset.store.method.
*Type: boolean* -auto.commit.interval.ms | C | 10 .. 86400000 | 60000 | high | [**LEGACY PROPERTY:** This setting is used by the simple legacy consumer only. When using the high-level KafkaConsumer, the global `auto.commit.interval.ms` property must be used instead]. The frequency in milliseconds that the consumer offsets are committed (written) to offset storage.
*Type: integer* +auto.commit.interval.ms | C | 10 .. 86400000 | 60000 | high | [**LEGACY PROPERTY:** This setting is used by the simple legacy consumer only. When using the high-level KafkaConsumer, the global `auto.commit.interval.ms` property must be used instead]. The frequency in milliseconds that the consumer offsets are committed (written) to offset storage. This property is ignored for share consumers.
*Type: integer* auto.offset.reset | C | smallest, earliest, beginning, largest, latest, end, error | largest | high | Action to take when there is no initial offset in offset store or the desired offset is out of range: 'smallest','earliest' - automatically reset the offset to the smallest offset, 'largest','latest' - automatically reset the offset to the largest offset, 'error' - trigger an error (ERR__AUTO_OFFSET_RESET) which is retrieved by consuming messages and checking 'message->err'.
*Type: enum value* offset.store.path | C | | . | low | **DEPRECATED** Path to local file for storing offsets. If the path is a directory a filename will be automatically generated in that directory based on the topic and partition. File-based offset storage will be removed in a future version.
*Type: string* offset.store.sync.interval.ms | C | -1 .. 86400000 | -1 | low | **DEPRECATED** fsync() interval for the offset file, in milliseconds. Use -1 to disable syncing, and 0 for immediate sync after each write. File-based offset storage will be removed in a future version.
*Type: integer* offset.store.method | C | file, broker | broker | low | **DEPRECATED** Offset commit store method: 'file' - DEPRECATED: local file store (offset.store.path, et.al), 'broker' - broker commit store (requires "group.id" to be configured and Apache Kafka 0.8.2 or later on the broker.).
*Type: enum value* -consume.callback.max.messages | C | 0 .. 1000000 | 0 | low | Maximum number of messages to dispatch in one `rd_kafka_consume_callback*()` call (0 = unlimited)
*Type: integer* +consume.callback.max.messages | C | 0 .. 1000000 | 0 | low | Maximum number of messages to dispatch in one `rd_kafka_consume_callback*()` call (0 = unlimited). This property is not supported for share consumers.
*Type: integer* ### C/P legend: C = Consumer, P = Producer, * = both diff --git a/lib/librdkafka-2.10.1/CONTRIBUTING.md b/lib/librdkafka-2.15.0/CONTRIBUTING.md similarity index 100% rename from lib/librdkafka-2.10.1/CONTRIBUTING.md rename to lib/librdkafka-2.15.0/CONTRIBUTING.md diff --git a/lib/librdkafka-2.10.1/Doxyfile b/lib/librdkafka-2.15.0/Doxyfile similarity index 100% rename from lib/librdkafka-2.10.1/Doxyfile rename to lib/librdkafka-2.15.0/Doxyfile diff --git a/lib/librdkafka-2.10.1/INTRODUCTION.md b/lib/librdkafka-2.15.0/INTRODUCTION.md similarity index 70% rename from lib/librdkafka-2.10.1/INTRODUCTION.md rename to lib/librdkafka-2.15.0/INTRODUCTION.md index 5b559380341..2fde8ac51fa 100644 --- a/lib/librdkafka-2.10.1/INTRODUCTION.md +++ b/lib/librdkafka-2.15.0/INTRODUCTION.md @@ -10,97 +10,120 @@ librdkafka also provides a native C++ interface. **Table of Contents** - [Introduction to librdkafka - the Apache Kafka C/C++ client library](#introduction-to-librdkafka---the-apache-kafka-cc-client-library) - - [Performance](#performance) - - [High throughput](#high-throughput) - - [Low latency](#low-latency) - - [Latency measurement](#latency-measurement) - - [Compression](#compression) - - [Message reliability](#message-reliability) - - [Producer message delivery success](#producer-message-delivery-success) - - [Producer message delivery failure](#producer-message-delivery-failure) - - [Error: Timed out in transmission queue](#error-timed-out-in-transmission-queue) - - [Error: Timed out in flight to/from broker](#error-timed-out-in-flight-tofrom-broker) - - [Error: Temporary broker-side error](#error-temporary-broker-side-error) - - [Error: Temporary errors due to stale metadata](#error-temporary-errors-due-to-stale-metadata) - - [Error: Local time out](#error-local-time-out) - - [Error: Permanent errors](#error-permanent-errors) - - [Producer retries](#producer-retries) - - [Reordering](#reordering) - - [Idempotent Producer](#idempotent-producer) - - [Guarantees](#guarantees) - - [Ordering and message sequence numbers](#ordering-and-message-sequence-numbers) - - [Partitioner considerations](#partitioner-considerations) - - [Message timeout considerations](#message-timeout-considerations) - - [Leader change](#leader-change) - - [Error handling](#error-handling) - - RD_KAFKA_RESP_ERR_OUT_OF_ORDER_SEQUENCE_NUMBER - - RD_KAFKA_RESP_ERR_DUPLICATE_SEQUENCE_NUMBER - - RD_KAFKA_RESP_ERR_UNKNOWN_PRODUCER_ID - - [Standard errors](#standard-errors) - - [Message persistence status](#message-persistence-status) - - [Transactional Producer](#transactional-producer) - - [Error handling](#error-handling-1) - - [Old producer fencing](#old-producer-fencing) - - [Configuration considerations](#configuration-considerations) - - [Exactly Once Semantics (EOS) and transactions](#exactly-once-semantics-eos-and-transactions) - - [Usage](#usage) - - [Documentation](#documentation) - - [Initialization](#initialization) - - [Configuration](#configuration) - - [Example](#example) - - [Termination](#termination) - - [High-level KafkaConsumer](#high-level-kafkaconsumer) - - [Producer](#producer) - - [Admin API client](#admin-api-client) - - [Speeding up termination](#speeding-up-termination) - - [Threads and callbacks](#threads-and-callbacks) - - [Brokers](#brokers) - - [SSL](#ssl) - - [OAUTHBEARER with support for OIDC](#oauthbearer-with-support-for-oidc) - - [Sparse connections](#sparse-connections) - - [Random broker selection](#random-broker-selection) - - [Persistent broker connections](#persistent-broker-connections) - - [Connection close](#connection-close) - - [Fetch From Follower](#fetch-from-follower) - - [Logging](#logging) - - [Debug contexts](#debug-contexts) - - [Feature discovery](#feature-discovery) - - [Producer API](#producer-api) - - [Simple Consumer API (legacy)](#simple-consumer-api-legacy) - - [Offset management](#offset-management) - - [Auto offset commit](#auto-offset-commit) - - [At-least-once processing](#at-least-once-processing) - - [Auto offset reset](#auto-offset-reset) - - [Consumer groups](#consumer-groups) - - [Static consumer groups](#static-consumer-groups) - - [Next generation of the consumer group protocol: KIP 848](#next-generation-of-the-consumer-group-protocol-kip-848) - - [Note on Batch consume APIs](#note-on-batch-consume-apis) - - [Topics](#topics) - - [Unknown or unauthorized topics](#unknown-or-unauthorized-topics) - - [Topic metadata propagation for newly created topics](#topic-metadata-propagation-for-newly-created-topics) - - [Topic auto creation](#topic-auto-creation) - - [Metadata](#metadata) - - [\< 0.9.3](#lt093) - - [\> 0.9.3](#gt093-1) - - [Query reasons](#query-reasons) - - [Caching strategy](#caching-strategy) - - [Fatal errors](#fatal-errors) - - [Fatal producer errors](#fatal-producer-errors) - - [Fatal consumer errors](#fatal-consumer-errors) - - [Compatibility](#compatibility) - - [Broker version compatibility](#broker-version-compatibility) - - [Broker version \>= 0.10.0.0 (or trunk)](#broker-version--01000-or-trunk) - - [Broker versions 0.9.0.x](#broker-versions-090x) - - [Broker versions 0.8.x.y](#broker-versions-08xy) - - [Detailed description](#detailed-description) - - [Supported KIPs](#supported-kips) - - [Supported protocol versions](#supported-protocol-versions) + - [Performance](#performance) + - [High throughput](#high-throughput) + - [Low latency](#low-latency) + - [Latency measurement](#latency-measurement) + - [Compression](#compression) + - [Message reliability](#message-reliability) + - [Producer message delivery success](#producer-message-delivery-success) + - [Producer message delivery failure](#producer-message-delivery-failure) + - [Error: Timed out in transmission queue](#error-timed-out-in-transmission-queue) + - [Error: Timed out in flight to/from broker](#error-timed-out-in-flight-tofrom-broker) + - [Error: Temporary broker-side error](#error-temporary-broker-side-error) + - [Error: Temporary errors due to stale metadata](#error-temporary-errors-due-to-stale-metadata) + - [Error: Local time out](#error-local-time-out) + - [Error: Permanent errors](#error-permanent-errors) + - [Producer retries](#producer-retries) + - [Reordering](#reordering) + - [Idempotent Producer](#idempotent-producer) + - [Guarantees](#guarantees) + - [Ordering and message sequence numbers](#ordering-and-message-sequence-numbers) + - [Partitioner considerations](#partitioner-considerations) + - [Message timeout considerations](#message-timeout-considerations) + - [Leader change](#leader-change) + - [Error handling](#error-handling) + - RD_KAFKA_RESP_ERR_OUT_OF_ORDER_SEQUENCE_NUMBER + - RD_KAFKA_RESP_ERR_DUPLICATE_SEQUENCE_NUMBER + - RD_KAFKA_RESP_ERR_UNKNOWN_PRODUCER_ID + - [Standard errors](#standard-errors) + - [Message persistence status](#message-persistence-status) + - [Transactional Producer](#transactional-producer) + - [Error handling](#error-handling-1) + - [Old producer fencing](#old-producer-fencing) + - [Configuration considerations](#configuration-considerations) + - [Exactly Once Semantics (EOS) and transactions](#exactly-once-semantics-eos-and-transactions) + - [Usage](#usage) + - [Documentation](#documentation) + - [Initialization](#initialization) + - [Configuration](#configuration) + - [Example](#example) + - [Termination](#termination) + - [High-level KafkaConsumer](#high-level-kafkaconsumer) + - [Producer](#producer) + - [Admin API client](#admin-api-client) + - [Speeding up termination](#speeding-up-termination) + - [Threads and callbacks](#threads-and-callbacks) + - [Brokers](#brokers) + - [SSL](#ssl) + - [OAUTHBEARER with support for OIDC](#oauthbearer-with-support-for-oidc) + - [JWT bearer grant type (KIP-1139)](#jwt-bearer-grant-type-kip-1139) + - [Metadata based authentication](#metadata-based-authentication) + - [Azure IMDS](#azure-imds) + - [Sparse connections](#sparse-connections) + - [Random broker selection](#random-broker-selection) + - [Persistent broker connections](#persistent-broker-connections) + - [Connection close](#connection-close) + - [Fetch From Follower](#fetch-from-follower) + - [Logging](#logging) + - [Debug contexts](#debug-contexts) + - [Feature discovery](#feature-discovery) + - [Producer API](#producer-api) + - [Simple Consumer API (legacy)](#simple-consumer-api-legacy) + - [Offset management](#offset-management) + - [Auto offset commit](#auto-offset-commit) + - [At-least-once processing](#at-least-once-processing) + - [Auto offset reset](#auto-offset-reset) + - [Consumer groups](#consumer-groups) + - [Static consumer groups](#static-consumer-groups) + - [Next Generation Consumer Group Protocol (KIP-848)](#next-generation-consumer-group-protocol-kip-848) + - [Overview](#overview) + - [Available Features](#available-features) + - [Contract Changes](#contract-changes) + - [Client Configuration changes](#client-configuration-changes) + - [Rebalance Callback Changes](#rebalance-callback-changes) + - [Regex Subscription Changes](#regex-subscription-changes) + - [Static Group Membership](#static-group-membership) + - [Session Timeout \& Fetching](#session-timeout--fetching) + - [Closing / Auto-Commit](#closing--auto-commit) + - [Error Handling Changes](#error-handling-changes) + - [Summary of Key Differences (Classic vs Next-Gen)](#summary-of-key-differences-classic-vs-next-gen) + - [Minimal Example Config](#minimal-example-config) + - [Classic Protocol](#classic-protocol) + - [Next-Gen Protocol / KIP-848](#next-gen-protocol--kip-848) + - [Rebalance Callback Migration](#rebalance-callback-migration) + - [Range Assignor (Classic)](#range-assignor-classic) + - [Incremental Assignor (Including Range in Consumer / KIP-848, Any Protocol)](#incremental-assignor-including-range-in-consumer--kip-848-any-protocol) + - [Upgrade and Downgrade](#upgrade-and-downgrade) + - [Migration Checklist (Next-Gen Protocol / KIP-848)](#migration-checklist-next-gen-protocol--kip-848) + - [Note on Batch consume APIs](#note-on-batch-consume-apis) + - [Share consumers (Queues for Kafka)](#share-consumers-queues-for-kafka) + - [Topics](#topics) + - [Unknown or unauthorized topics](#unknown-or-unauthorized-topics) + - [Topic metadata propagation for newly created topics](#topic-metadata-propagation-for-newly-created-topics) + - [Topic auto creation](#topic-auto-creation) + - [Metadata](#metadata) + - [\< 0.9.3](#lt093) + - [\> 0.9.3](#gt093-1) + - [Query reasons](#query-reasons) + - [Caching strategy](#caching-strategy) + - [Fatal errors](#fatal-errors) + - [Fatal producer errors](#fatal-producer-errors) + - [Fatal consumer errors](#fatal-consumer-errors) + - [Compatibility](#compatibility) + - [Broker version compatibility](#broker-version-compatibility) + - [Broker version \>= 0.10.0.0 (or trunk)](#broker-version--01000-or-trunk) + - [Broker versions 0.9.0.x](#broker-versions-090x) + - [Broker versions 0.8.x.y](#broker-versions-08xy) + - [Detailed description](#detailed-description) + - [Supported KIPs](#supported-kips) + - [Supported protocol versions](#supported-protocol-versions) - [Recommendations for language binding developers](#recommendations-for-language-binding-developers) - - [Expose the configuration interface pass-thru](#expose-the-configuration-interface-pass-thru) - - [Error constants](#error-constants) - - [Reporting client software name and version to broker](#reporting-client-software-name-and-version-to-broker) - - [Documentation reuse](#documentation-reuse) - - [Community support](#community-support) + - [Expose the configuration interface pass-thru](#expose-the-configuration-interface-pass-thru) + - [Error constants](#error-constants) + - [Reporting client software name and version to broker](#reporting-client-software-name-and-version-to-broker) + - [Documentation reuse](#documentation-reuse) + - [Community support](#community-support) @@ -1197,6 +1220,79 @@ To use this authentication method the client needs to be configured as follows: provided to the broker. A comma-separated list of key=value pairs. For example: `supportFeatureX=true,organizationId=sales-emea` +* `https.ca.location` - (optional) to customize the CA certificates + location. + +* `https.ca.pem` - (optional) to provide the CA certificates as a PEM string. + + +##### JWT bearer grant type (KIP-1139) + +This KIP adds support for the `client_credentials, urn:ietf:params:oauth:grant-type:jwt-bearer` +grant type, with a series of properties to be used for creating a JWT assertion +sent to the token endpoint. The authenticated principal corresponds to the +`sub` claim returned by token endpoint, `sasl.oauthbearer.client.id` and +`sasl.oauthbearer.client.secret` aren't used. Required JWT claims must be set +either through the template or with the `claim` properties. + +* `sasl.oauthbearer.grant.type` - changes the default grant type, set it to + `urn:ietf:params:oauth:grant-type:jwt-bearer`. +* `sasl.oauthbearer.assertion.algorithm` - JWT algorithm defaults to `RS256`. +* `sasl.oauthbearer.assertion.private.key.file` - a private key file for signing + the token. +* `sasl.oauthbearer.assertion.private.key.passphrase` - (optional) passphrase for the key if encrypted. +* `sasl.oauthbearer.assertion.private.key.pem` - alternatively to the key file + it's possible to pass the private key as a string. +* `sasl.oauthbearer.assertion.file` - (optional) assertion file: with this property all other + assertion related fields are ignored and the assertion is read from this file + that should be periodically updated. +* `sasl.oauthbearer.assertion.jwt.template.file` - (optional) template file: a template containing + a default `header` and `payload` that can be overwritten by the `claim` properties. +* `sasl.oauthbearer.assertion.claim.aud`, + `sasl.oauthbearer.assertion.claim.exp.seconds`, + `sasl.oauthbearer.assertion.claim.iss`, + `sasl.oauthbearer.assertion.claim.jti.include`, + `sasl.oauthbearer.assertion.claim.sub` - (optional) the `claim` properties: + it's possible to dynamically customize the JWT claims with these or to + skip the template file and use only these properties. + + +##### Metadata based authentication + +Some cloud providers added the ability to authenticate clients based on +OAUTHBEARER/OIDC tokens returned from endpoints that can only be called from +a given instance. Such endpoints are served on a specific IP address (169.254.169.254) +that is a link-local metadata endpoint. + + +While there is no standard for that still, librdkafka has support for +some metadata based OAUTHBEARER authentication types. + + +Currently these authentication types are supported: + + +###### Azure IMDS + +To use this method you set: + +* `sasl.oauthbearer.metadata.authentication.type=azure_imds` this makes it so + that ` sasl.oauthbearer.client.id` and `sasl.oauthbearer.client.secret` + aren't required. +* `sasl.oauthbearer.config` is a general purpose configuration property + In this case it accepts comma-separated `key=value` pairs. + The `query` key is required in case `sasl.oauthbearer.token.endpoint.url` isn't + specified and its value is the GET query string to append + to the token endpoint URL. Such query string contains params required by + Azure IMDS such as `client_id` (the UAMI), `resource` for determining the + target audience and `api-version` for the API version to be used by the endpoint +* `sasl.oauthbearer.token.endpoint.url` (optional) is set automatically. + when choosing `sasl.oauthbearer.metadata.authentication.type=azure_imds` but can + be customized. + + +_Example:_ `sasl.oauthbearer.metadata.authentication.type=azure_imds` and +`sasl.oauthbearer.config=params=api-version=2025-04-07&resource=api://&client_id=` @@ -1304,40 +1400,40 @@ on the broker. Extensive debugging of librdkafka can be enabled by setting the `debug` configuration property to a CSV string of debug contexts: -Debug context | Type | Description ---------------|----------|---------------------- -generic | * | General client instance level debugging. Includes initialization and termination debugging. -broker | * | Broker and connection state debugging. -topic | * | Topic and partition state debugging. Includes leader changes. -metadata | * | Cluster and topic metadata retrieval debugging. -feature | * | Kafka protocol feature support as negotiated with the broker. -queue | producer | Message queue debugging. -msg | * | Message debugging. Includes information about batching, compression, sizes, etc. -protocol | * | Kafka protocol request/response debugging. Includes latency (rtt) printouts. -cgrp | consumer | Low-level consumer group state debugging. -security | * | Security and authentication debugging. -fetch | consumer | Consumer message fetch debugging. Includes decision when and why messages are fetched. -interceptor | * | Interceptor interface debugging. -plugin | * | Plugin loading debugging. -consumer | consumer | High-level consumer debugging. -admin | admin | Admin API debugging. -eos | producer | Idempotent Producer debugging. -mock | * | Mock cluster functionality debugging. -assignor | consumer | Detailed consumer group partition assignor debugging. -conf | * | Display set configuration properties on startup. -all | * | All of the above. +| Debug context | Type | Description | +| ------------- | -------- | ------------------------------------------------------------------------------------------- | +| generic | * | General client instance level debugging. Includes initialization and termination debugging. | +| broker | * | Broker and connection state debugging. | +| topic | * | Topic and partition state debugging. Includes leader changes. | +| metadata | * | Cluster and topic metadata retrieval debugging. | +| feature | * | Kafka protocol feature support as negotiated with the broker. | +| queue | producer | Message queue debugging. | +| msg | * | Message debugging. Includes information about batching, compression, sizes, etc. | +| protocol | * | Kafka protocol request/response debugging. Includes latency (rtt) printouts. | +| cgrp | consumer | Low-level consumer group state debugging. | +| security | * | Security and authentication debugging. | +| fetch | consumer | Consumer message fetch debugging. Includes decision when and why messages are fetched. | +| interceptor | * | Interceptor interface debugging. | +| plugin | * | Plugin loading debugging. | +| consumer | consumer | High-level consumer debugging. | +| admin | admin | Admin API debugging. | +| eos | producer | Idempotent Producer debugging. | +| mock | * | Mock cluster functionality debugging. | +| assignor | consumer | Detailed consumer group partition assignor debugging. | +| conf | * | Display set configuration properties on startup. | +| all | * | All of the above. | Suggested debugging settings for troubleshooting: -Problem space | Type | Debug setting ------------------------|----------|------------------- -Producer not delivering messages to broker | producer | `broker,topic,msg` -Consumer not fetching messages | consumer | Start with `consumer`, or use `cgrp,fetch` for detailed information. -Consumer starts reading at unexpected offset | consumer | `consumer` or `cgrp,fetch` -Authentication or connectivity issues | * | `broker,auth` -Protocol handling or latency | * | `broker,protocol` -Topic leader and state | * | `topic,metadata` +| Problem space | Type | Debug setting | +| -------------------------------------------- | -------- | -------------------------------------------------------------------- | +| Producer not delivering messages to broker | producer | `broker,topic,msg` | +| Consumer not fetching messages | consumer | Start with `consumer`, or use `cgrp,fetch` for detailed information. | +| Consumer starts reading at unexpected offset | consumer | `consumer` or `cgrp,fetch` | +| Authentication or connectivity issues | * | `broker,auth` | +| Protocol handling or latency | * | `broker,protocol` | +| Topic leader and state | * | `topic,metadata` | @@ -1602,92 +1698,225 @@ the original fatal error code and reason. To read more about static group membership, see [KIP-345](https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances). - -### Next generation of the consumer group protocol: [KIP 848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) + +### Next Generation Consumer Group Protocol ([KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol)) -Starting from librdkafka 2.4.0 the next generation consumer group rebalance protocol -defined in KIP 848 is introduced. +Starting with **librdkafka v2.12.0** (GA release), the next generation consumer group rebalance protocol defined in **[KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol)** is **production-ready**. -**Warning** -It's still in **Preview** which means it's _not production-ready_, -given it's still under validation. -Features and their contract might change in future. +**Note:** The new consumer group protocol defined in [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) is not enabled by default. There are few contract change associated with the new protocol and might cause breaking changes. `group.protocol` configuration property dictates whether to use the new `consumer` protocol or older `classic` protocol. It defaults to `classic` if not provided. -With this protocol the role of the Group Leader (a member) is removed and -the assignment is calculated by the Group Coordinator (a broker) and sent -to each member through heartbeats. + +#### Overview +- **What changed:** + The **Group Leader role** (consumer member) is removed. Assignments are calculated by the **Group Coordinator (broker)** and distributed via **heartbeats**. -To test it, a Kafka cluster must be set up with broker version 4.0.0 or newer. +- **Requirements:** + - Broker version: **v4.0.0+** + - librdkafka version: **v2.12.0+**: GA (production-ready) -Client side, it can be enabled by setting the new property `group.protocol=consumer`. -A second property named `group.remote.assignor` is added to choose desired -remote assignor. +- **Enablement (client-side):** + - `group.protocol=consumer` + - `group.remote.assignor=` (optional; broker-controlled if `NULL`; default broker assignor is **`uniform`**) -**Available features** + +#### Available Features -All the features described in the KIP are available. +All [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) features are supported including: -- Subscription to one or more topics -- Rebalance callbacks (see contract changes) +- Subscription to one or more topics, including **regular expression (regex) subscriptions** +- Rebalance callbacks (**incremental only**) - Static group membership -- Configure remote assignor -- Max poll interval is enforced -- Offline upgrade from an empty consumer group with committed offsets -- Regular expression support when subscribing -- AdminClient changes as described in the KIP - -**Contract changes** - -Along with the new feature there are some needed contract changes, -so the protocol will be enabled by default only with a librdkafka major release. - -- Deprecated client configurations with the new protocol: - - `partition.assignment.strategy` replaced by `group.remote.assignor` - - `session.timeout.ms` replaced by broker configuration `group.consumer.session.timeout.ms` - - `heartbeat.interval.ms`, replaced by broker configuration `group.consumer.heartbeat.interval.ms` - - `group.protocol.type` which is not used in the new protocol - -- Protocol rebalance is fully incremental, so the only allowed functions to - use in a rebalance callback will be `rd_kafka_incremental_assign` and - `rd_kafka_incremental_unassign`. In the `classic` protocol, the expected - function to call is determined based on the chosen `partition.assignment.strategy` - but this is removed for the `consumer` protocol. - - When setting the `group.remote.assignor` property, it's already - required to use the incremental assign and unassign functions. - All assignors are sticky with new protocol, including the _range_ one, that wasn't. - -- With a static group membership, if two members are using the same - `group.instance.id`, the one that joins the consumer group later will be - fenced, with the fatal `UNRELEASED_INSTANCE_ID` error. Before, it was the existing - member to be fenced. This was changed to avoid two members contending the - same id. It also means that any instance that crashes won't be automatically - replaced by a new instance until session times out and it's especially required - to check that consumers are being closed properly on shutdown. Ensuring that - no two instances with same `group.instance.id` are running at any time - is also important. - -- Session timeout is remote only and, if the Coordinator isn't reachable - by a member, this will continue to fetch messages, even if it won't be able to - commit them. Otherwise, the member will be fenced as soon as it receives an - heartbeat response from the Coordinator. - With `classic` protocol, instead, member stops fetching when session timeout - expires on the client. - - For the same reason, when closing or unsubscribing with auto-commit set, - the member will try to commit until a specific timeout has passed. - Currently the timeout is a fixed one and corresponds to the default remote session - timeout in Apache Kafka but it'll be possible to pass a custom timeout - when calling close in future with [KIP-1092](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=321719077). - -- An `UNKNOWN_TOPIC_OR_PART` error isn't received anymore when a consumer is - subscribing to a topic that doesn't exist in local cache, as the consumer - is still subscribing to the topic and it could be created just after that. - -- A TOPIC_AUTHORIZATION_FAILED error will be returned as was happening - with group protocol classic, it happens once per heartbeat or subscription change and - it's a single error for the whole subscription. It's reported even when only a single topic - isn't authorized. +- Configurable remote assignor +- Enforced max poll interval +- Upgrade from `classic` protocol or downgrade from `consumer` protocol +- AdminClient changes as per KIP + + +#### Contract Changes + + +##### Client Configuration changes + +| Classic Protocol (Deprecated Configs in KIP-848) | KIP-848 / Next-Gen Replacement | +| ------------------------------------------------ | ----------------------------------------------------- | +| `partition.assignment.strategy` | `group.remote.assignor` | +| `session.timeout.ms` | Broker config: `group.consumer.session.timeout.ms` | +| `heartbeat.interval.ms` | Broker config: `group.consumer.heartbeat.interval.ms` | +| `group.protocol.type` | Not used in the new protocol | + +**Note:** The properties listed under “Classic Protocol (Deprecated Configs in KIP-848)” are **no longer used** when using the KIP-848 consumer protocol. + + +##### Regex Subscription Changes + +Regex matching in the `consumer` protocol is performed on the broker side, using the **Google RE2/J** regex engine. +This differs from the `classic` protocol—where librdkafka and derived clients performed regex evaluation locally using the **libc regex** engine. + +As part of adopting the `consumer` protocol, librdkafka (and derived clients) now rely on the broker’s RE2/J engine for regex-based subscriptions, effectively replacing the previous `libc`-based matching behavior. + +**A known case which would fail with the new protocol**: we have topics `topic-1` and `topic-2`. If we subscribe with `^topic` or `^topic*` in the `classic` protocol, it's assigned partitions from both topics but no partitions is assigned with the `consumer` protocol. That's because the broker side regex implementation as well as the Java classic protocol one requires that regexes match the complete topic, while `libc` one only finds the pattern into the topic, that could also be present as a prefix. To obtain the same result with this example you'd use `^topic.*` instead. + + +##### Rebalance Callback Changes + +- Protocol is **fully incremental**. +- **Inside the rebalance callback**, you **must use**: + - `rd_kafka_incremental_assign(rk, partitions)` to assign partitions + - `rd_kafka_incremental_unassign(rk, partitions)` to revoke partitions +- **Do not** use `rd_kafka_assign()` or other assignment APIs in KIP-848. +- **Important:** The `partitions` parameter passed to `rd_kafka_incremental_assign` or `rd_kafka_incremental_unassign` contains only an **incremental list of partitions**—those being added or revoked—rather than the full partition list returned by `rd_kafka_assign(rk, partitions)` in the **range assignor of the classic protocol**, which was the default. +- All assignors are **sticky**, including `range` (which wasn’t sticky before). + + +##### Static Group Membership + +- Duplicate `group.instance.id` handling: + - **Newly joining member** is fenced with **UNRELEASED_INSTANCE_ID (fatal)**. + - (Classic protocol fenced the **existing** member instead.) +- Implications: + - Ensure only **one active instance per `group.instance.id`**. + - Consumers must shut down cleanly to avoid blocking replacements until session timeout expires. + + +##### Session Timeout & Fetching + +- **Session timeout is broker-controlled**: + - If the Coordinator is unreachable, a consumer **continues fetching messages** but cannot commit offsets. + - Consumer is fenced once a heartbeat response is received from the Coordinator. +- In the classic protocol, the client stopped fetching when session timeout expired. + + +##### Closing / Auto-Commit + +- On `close()` or unsubscribe with auto-commit enabled: + - Member retries committing offsets until a timeout expires. + - Currently uses the **default remote session timeout**. + - Future **KIP-1092** will allow custom commit timeouts. + + +##### Error Handling Changes + +- `UNKNOWN_TOPIC_OR_PART` (**subscription case**): + - No longer returned if a topic is missing in the **local cache** when subscribing; the subscription proceeds. +- `TOPIC_AUTHORIZATION_FAILED`: + - Reported once per heartbeat or subscription change, even if only one topic is unauthorized. + + +##### Summary of Key Differences (Classic vs Next-Gen) + +- **Assignment:** Classic protocol calculated by **Group Leader (consumer)**; KIP-848 calculated by **Group Coordinator (broker)** +- **Assignors:** Classic range assignor was **not sticky**; KIP-848 assignors are **sticky**, including range +- **Deprecated configs:** Classic client configs are replaced by `group.remote.assignor` and broker-controlled session/heartbeat configs +- **Static membership fencing:** KIP-848 fences **new member** on duplicate `group.instance.id` +- **Session timeout:** Classic enforced on client; KIP-848 enforced on broker +- **Auto-commit on close:** Classic stops at client session timeout; KIP-848 retries until remote timeout +- **Unknown topics:** KIP-848 does not return error on subscription if topic missing +- **Upgrade/Downgrade:** KIP-848 supports upgrade/downgrade from/to `classic` and `consumer` protocols + + +#### Minimal Example Config + + +##### Classic Protocol +```properties +# Optional; default is 'classic' +group.protocol=classic + +partition.assignment.strategy= +session.timeout.ms=45000 +heartbeat.interval.ms=15000 +``` + + +##### Next-Gen Protocol / KIP-848 +```properties +group.protocol=consumer + +# Optional: select a remote assignor +# Valid options currently: 'uniform' or 'range' +# group.remote.assignor= +# If unset(NULL), broker chooses the assignor (default: 'uniform') + +# Session & heartbeat now controlled by broker: +# group.consumer.session.timeout.ms +# group.consumer.heartbeat.interval.ms +``` + + +#### Rebalance Callback Migration + + +##### Range Assignor (Classic) +```c +/* Rebalance callback for range assignor (classic) */ +static void rebalance_cb (rd_kafka_t *rk, + rd_kafka_resp_err_t err, + rd_kafka_topic_partition_list_t *partitions, + void *opaque) { + switch (err) { + case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: + rd_kafka_assign(rk, partitions); /* full partition list */ + break; + + case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: + rd_kafka_assign(rk, NULL); /* revoke all partitions */ + break; + + default: + fprintf(stderr, "Rebalance error: %s\n", rd_kafka_err2str(err)); + break; + } +} +``` + + +##### Incremental Assignor (Including Range in Consumer / KIP-848, Any Protocol) + +```c +/* Rebalance callback for incremental assignor */ +static void rebalance_cb (rd_kafka_t *rk, + rd_kafka_resp_err_t err, + rd_kafka_topic_partition_list_t *partitions, + void *opaque) { + switch (err) { + case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: + rd_kafka_incremental_assign(rk, partitions); /* incremental partitions only */ + break; + + case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: + rd_kafka_incremental_unassign(rk, partitions); + break; + + default: + fprintf(stderr, "Rebalance error: %s\n", rd_kafka_err2str(err)); + break; + } +} +``` +**Note:** +- The `partitions` list contains **only partitions being added or revoked**, not the full partition list as in the classic `rd_kafka_assign()`. + + +#### Upgrade and Downgrade + +- A group made up entirely of `classic` consumers runs under the classic protocol. +- The group is **upgraded to the consumer protocol** as soon as at least one `consumer` protocol member joins. +- The group is **downgraded back to the classic protocol** if the last `consumer` protocol member leaves while `classic` members remain. +- Both **rolling upgrade** (classic → consumer) and **rolling downgrade** (consumer → classic) are supported. + + + +#### Migration Checklist (Next-Gen Protocol / [KIP-848](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol)) + +1. Upgrade to **librdkafka ≥ v2.12.0** (GA release) +2. Run against **Kafka brokers ≥ v4.0.0** +3. Set `group.protocol=consumer` +4. Optionally set `group.remote.assignor`; leave `NULL` for broker-controlled (default: `uniform`), valid options: `uniform` or `range` +5. Replace deprecated configs with new ones +6. Update rebalance callbacks to **incremental APIs only** +7. Review static membership handling (`group.instance.id`) +8. Ensure proper shutdown to avoid fencing issues +9. Adjust error handling for unknown topics and authorization failures @@ -1711,6 +1940,264 @@ set `rebalance_cb` configuration property (refer [examples/rdkafka_complex_consu for the help with the usage) for the consumer. + +### Share consumers (Queues for Kafka) + +> **Preview feature.** The share consumer is provided as a preview. Its public +> interfaces (the `rd_kafka_share_*` APIs in [rdkafka.h](src/rdkafka.h)) may +> change in a future release and it is not recommended for production use. +> See [Current limitations](#share-consumer-current-limitations) below. + +Share groups ([KIP-932](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka)) +bring queue-like semantics to Kafka. Where a consumer group assigns +each partition to exactly one member at a time, a *share group* lets multiple +members consume from the **same** partitions cooperatively. The unit of +progress is the individual record rather than the committed offset: each +delivered record is *acquired* by a member under a time-limited acquisition +lock, processed, and then *acknowledged*. The lock duration is a broker/group +setting (`group.share.record.lock.duration.ms`, 30 seconds by default) and is +not configured on this client. A record that is not acknowledged before its +lock expires, or that is explicitly released, becomes available again and may +be redelivered — possibly to a different member. A record thus moves through the +states *available* → *acquired* → *acknowledged*; a rejected record, or one that +exceeds the broker's delivery-count limit, becomes *archived* and is no longer +delivered. This makes it possible to scale the number of consumers beyond the +number of partitions and to distribute work like a traditional queue. + +For a conceptual overview of share groups and Queues for Kafka, see the +[Confluent share consumer documentation](https://docs.confluent.io/platform/current/clients/share-consumers.html). + +A share consumer is a distinct handle type, `rd_kafka_share_t`, created with +`rd_kafka_share_consumer_new()` (not `rd_kafka_new()`). The full API reference +lives in the *Share consumer (Queues for Kafka)* section of +[rdkafka.h](src/rdkafka.h). The share consumer is currently available through +the **C API only**; there is no C++ (`rdkafkacpp.h`) wrapper yet. + +#### Broker requirement + +A share consumer requires a broker with **share groups enabled**. Share groups +are available since **Apache Kafka 4.2.0**. Partition assignment is entirely +broker-driven (via the share group heartbeat); there is no client-side +rebalance callback or `assign()` step. + +#### Lifecycle + +``` +rd_kafka_share_consumer_new() # create handle from an rd_kafka_conf_t +rd_kafka_share_subscribe() # subscribe to a set of topics + loop: + rd_kafka_share_poll() # fetch a batch of messages + ... process / acknowledge ... + rd_kafka_share_commit_*() # (optional) flush acknowledgements +rd_kafka_share_consumer_close() # send pending acks, leave the group +rd_kafka_share_destroy() # free the handle +``` + +#### Configuration + +Configure the handle through the normal `rd_kafka_conf_t` interface before +creating it. `group.id` is required. `share.acknowledgement.mode` is optional +and defaults to `implicit` (see below). It is the only share-specific client +property: other share-group settings (acquisition-lock duration, delivery-count +limit, session/heartbeat timeouts, isolation level, acquire mode and offset +reset) are broker/group settings and are not exposed by this client. + +Several regular-consumer properties do not apply to share consumers and are +rejected or ignored — for example `partition.assignment.strategy`, +`enable.auto.commit`, `auto.offset.reset` semantics, `isolation.level`, +`enable.partition.eof`, `group.instance.id`, `group.remote.assignor`, and the +per-partition fetch-queue tuning (`queued.min.messages`, +`queued.max.messages.kbytes`, `fetch.message.max.bytes`, etc.). A few network +defaults also differ for share consumers (for example `receive.message.max.bytes`, +`connections.max.idle.ms`, `reconnect.backoff.ms`). See +[CONFIGURATION.md](CONFIGURATION.md), where these properties are annotated, for +the authoritative list and the share-consumer default values. + +#### Polling and message batches + +`rd_kafka_share_poll()` returns a **batch** of messages in a single call +(unlike the regular single-message poll), as an opaque `rd_kafka_messages_t` +handle. Iterate it with `rd_kafka_messages_count()` and +`rd_kafka_messages_get()`, and release it with `rd_kafka_messages_destroy()` +(which is NULL-safe). `max.poll.records` (default 500) bounds the batch size. + +Record-level errors are surfaced as individual messages with a non-zero +`rd_kafka_message_t.err` field (the topic, partition and offset remain valid), +so the application should check each message's `err` before treating it as +data. + +#### Acknowledgement + +Every acquired record is acknowledged with one of three types: + +* **ACCEPT** (`RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT`) — processed + successfully. +* **RELEASE** (`RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE`) — not processed; make + the record available again for redelivery. +* **REJECT** (`RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT`) — do not deliver the + record again. + +**Note**: RENEW (acquisition-lock renewal) is not yet available; see +[Current limitations](#share-consumer-current-limitations). + +The number of times a record has already been delivered is available via +`rd_kafka_message_delivery_count()`, which is useful to detect and reject a +"poison" record after a threshold. The broker also enforces its own maximum +delivery count (`group.share.delivery.count.limit`, default 5); once a record +exceeds it the broker archives the record and stops redelivering it. + +There are two acknowledgement modes, selected by `share.acknowledgement.mode`: + +* **implicit** (default) — the application does not call the acknowledge APIs. + All records returned by a poll are automatically accepted (ACCEPT) on the + next `rd_kafka_share_poll()` / `rd_kafka_share_commit_sync()` / + `rd_kafka_share_commit_async()`. +* **explicit** — the application must acknowledge every record returned by a + poll (with `rd_kafka_share_acknowledge()`, + `rd_kafka_share_acknowledge_type()` or `rd_kafka_share_acknowledge_offset()`) + before the next poll. If any record from the previous batch is still + unacknowledged, `rd_kafka_share_poll()` returns + `RD_KAFKA_RESP_ERR__STATE`. Records you have acknowledged but not yet + committed are also committed when the consumer is closed (unless it is + destroyed with `RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE`). + +Acknowledgements are sent to the broker as part of the next poll, or flushed +explicitly with `rd_kafka_share_commit_async()` (fire-and-forget) or +`rd_kafka_share_commit_sync()` (blocks for broker replies, reporting +per-partition results). The acknowledgement-commit callback registered with +`rd_kafka_share_set_acknowledgement_commit_cb()` is invoked with the outcome +for each partition. + +#### Example: explicit acknowledgement + +The example below shows the full share-consumer lifecycle in explicit mode +(configure, create, subscribe, poll, acknowledge, commit, close, destroy) and +focuses on the per-record acknowledgement logic. It uses the real librdkafka +APIs but is not a complete program: call-level error handling (the +`rd_kafka_error_t` returned by `rd_kafka_share_poll()`, the acknowledge calls +and the commit calls, and the return value of `rd_kafka_conf_set()`) is omitted +for brevity. + +```c +char errstr[512]; + +rd_kafka_conf_t *conf = rd_kafka_conf_new(); +rd_kafka_conf_set(conf, "bootstrap.servers", "localhost:9092", + errstr, sizeof(errstr)); +rd_kafka_conf_set(conf, "group.id", "my-share-group", errstr, sizeof(errstr)); +rd_kafka_conf_set(conf, "share.acknowledgement.mode", "explicit", + errstr, sizeof(errstr)); + +rd_kafka_share_t *rkshare = + rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + +rd_kafka_share_subscribe(rkshare, topics); + +while (run) { + rd_kafka_messages_t *batch = NULL; + + rd_kafka_share_poll(rkshare, timeout_ms, &batch); + + size_t cnt = rd_kafka_messages_count(batch); + for (size_t i = 0; i < cnt; i++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, i); + + if (rkm->err) { + /* Records carrying a record-level error (rkm->err set) + * have already been acknowledged internally by + * librdkafka: RELEASEd for decompression failures, + * REJECTed for corrupt/unsupported batches. The + * application can re-acknowledge them here if required + * (e.g. to override the internal decision); otherwise + * it only acknowledges the records it received. */ + continue; + } + + switch (process(rkm)) { + case PROCESS_OK: + /* Processed successfully. */ + rd_kafka_share_acknowledge_type( + rkshare, rkm, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + break; + case PROCESS_TEMPORARY_FAILURE: + /* Release for redelivery (possibly to another member). */ + rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + break; + case PROCESS_PERMANENT_FAILURE: + /* Reject a "poison" record so it is not redelivered. */ + rd_kafka_share_acknowledge_type( + rkshare, rkm, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + break; + } + } + rd_kafka_messages_destroy(batch); + + /* Flush the acknowledgements; the next poll would also send them. + * Use rd_kafka_share_commit_sync(rkshare, timeout_ms, &partitions) + * to block and inspect the per-partition result. */ + rd_kafka_share_commit_async(rkshare); +} + +rd_kafka_share_consumer_close(rkshare); +rd_kafka_share_destroy(rkshare); +``` + +Complete runnable programs are in the examples directory: +[share_consumer.c](examples/share_consumer.c), +[share_consumer_commit_sync.c](examples/share_consumer_commit_sync.c) and +[share_consumer_commit_async.c](examples/share_consumer_commit_async.c). + +#### Thread safety + +The share consumer handle is **not thread-safe by design**: a single +`rd_kafka_share_t` handle must not be used concurrently from multiple threads. +This follows the share consumer design in +[KIP-932](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka), +where the share consumer — like the regular consumer — is single-threaded and +the application owns the threading model (typically one handle per thread, or +serialised access to a handle). Concurrent use is detected on a best-effort +basis and rejected with `RD_KAFKA_RESP_ERR__CONFLICT`. There is no wakeup +mechanism in this preview, so a blocking poll/commit ends only when its timeout +expires. + + +#### Current limitations (preview) + +The preview differs from the Apache Kafka Java share consumer in a number of +ways. The most important for application authors: + +* **Record limit is a soft bound.** `max.poll.records` (default 500) does not + strictly bound the number of records returned per poll (the strict-limit and + acquire-mode work of KIP-1206 is not implemented). +* **Acknowledgement types are limited to ACCEPT / RELEASE / REJECT.** There is + no acquisition-lock renewal (KIP-1222), so a long-running handler cannot + extend a record's lock and the record may be redelivered. +* **No wakeup API.** A blocking poll/commit can only be ended by its timeout. +* **`close()` has no timeout argument.** `rd_kafka_share_consumer_close()` + takes no timeout argument; close is bounded internally to roughly + `socket.timeout.ms`. +* **No deserialization in the C core**, hence no automatic RELEASE on a + deserialization failure. +* **No share-group admin operations** (describe/list/alter/delete share + groups and offsets) and **no share-consumer client-metrics APIs**. +* **No auto topic creation.** Share topics are resolved by topic id; + `allow.auto.create.topics` has no effect. +* **Failed acknowledgements are not retried** automatically; the error is + reported through the acknowledgement-commit callback or the + `rd_kafka_share_commit_sync()` per-partition results. +* **Some error codes differ** because librdkafka generates certain failures + locally without a broker round-trip (for example + `RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER`, `RD_KAFKA_RESP_ERR__TRANSPORT`, + `RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH`). Applications that branch on + error codes should account for this. +* **Single-broker fetch per poll.** Each poll fetches from one broker + (round-robin over the assigned partitions); other brokers' partitions are + served on subsequent polls. This is a throughput/latency detail, not a + correctness one — all brokers are served across successive polls. + + ### Topics @@ -1768,6 +2255,8 @@ be marked as non-existent and remain non-existent until a topic metadata refresh sees the topic again (after the topic has been re-created). +**Note**: `RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC*` during a `subscribe()` call occurs **only with the classic protocol**. With the next-gen `consumer` protocol (KIP-848), subscription proceeds even if the topic is not yet in the local cache (e.g., it may be created later). + #### Topic auto creation @@ -2019,7 +2508,7 @@ The [Apache Kafka Implementation Proposals (KIPs)](https://cwiki.apache.org/conf | KIP | Kafka release | Status | -|--------------------------------------------------------------------------|-----------------------------|-----------------------------------------------------------------------------------------------| +| ------------------------------------------------------------------------ | --------------------------- | --------------------------------------------------------------------------------------------- | | KIP-1 - Stop accepting request.required.acks > 1 | 0.9.0.0 | Not enforced on client (due to backwards compat with brokers <0.8.3) | | KIP-4 - Metadata protocol changes | 0.9.0.0, 0.10.0.0, 0.10.1.0 | Supported | | KIP-8 - Producer flush() | 0.9.0.0 | Supported | @@ -2132,10 +2621,13 @@ The [Apache Kafka Implementation Proposals (KIPs)](https://cwiki.apache.org/conf | KIP-735 - Increase default consumer session timeout | 3.0.0 | Supported | | KIP-768 - SASL/OAUTHBEARER OIDC support | 3.0 | Supported | | KIP-881 - Rack-aware Partition Assignment for Kafka Consumers | 3.5.0 | Supported | -| KIP-848 - The Next Generation of the Consumer Rebalance Protocol | 4.0.0 | Preview | +| KIP-848 - The Next Generation of the Consumer Rebalance Protocol | 4.0.0 | Supported | | KIP-899 - Allow producer and consumer clients to rebootstrap | 3.8.0 | Supported | | KIP-951 - Leader discovery optimisations for the client | 3.7.0 | Supported | | KIP-1082 - Require Client-Generated IDs over the ConsumerGroupHeartbeat | 4.0.0 | Supported | +| KIP-1102 - Enable clients to rebootstrap based on timeout or error code | 4.0.0 | Supported | +| KIP-1139 - Add support for OAuth jwt-bearer grant type | 4.1.0 (WIP) | Supported | +| KIP-932 - Queues for Kafka (share groups / share consumer) | 4.2.0 | Preview (C API; see the [Share consumers](#share-consumers-queues-for-kafka) usage section and the Share consumer (Queues for Kafka) section in [rdkafka.h](src/rdkafka.h)) | @@ -2148,49 +2640,49 @@ The [Apache Kafka Implementation Proposals (KIPs)](https://cwiki.apache.org/conf release of librdkafka. -| ApiKey | Request name | Kafka max | librdkafka max | -| ------- | ----------------------------- | ---------- | -------------- | -| 0 | Produce | 12 | 10 | -| 1 | Fetch | 17 | 16 | -| 2 | ListOffsets | 10 | 7 | -| 3 | Metadata | 13 | 12 | -| 8 | OffsetCommit | 9 | 9 | -| 9 | OffsetFetch | 9 | 9 | -| 10 | FindCoordinator | 6 | 2 | -| 11 | JoinGroup | 9 | 5 | -| 12 | Heartbeat | 4 | 3 | -| 13 | LeaveGroup | 5 | 1 | -| 14 | SyncGroup | 5 | 3 | -| 15 | DescribeGroups | 6 | 4 | -| 16 | ListGroups | 5 | 4 | -| 17 | SaslHandshake | 1 | 1 | -| 18 | ApiVersions | 4 | 3 | -| 19 | CreateTopics | 7 | 4 | -| 20 | DeleteTopics | 6 | 1 | -| 21 | DeleteRecords | 2 | 1 | -| 22 | InitProducerId | 5 | 4 | -| 23 | OffsetForLeaderEpoch | 4 | 2 | -| 24 | AddPartitionsToTxn | 5 | 0 | -| 25 | AddOffsetsToTxn | 4 | 0 | -| 26 | EndTxn | 5 | 1 | -| 28 | TxnOffsetCommit | 5 | 3 | -| 29 | DescribeAcls | 3 | 1 | -| 30 | CreateAcls | 3 | 1 | -| 31 | DeleteAcls | 3 | 1 | -| 32 | DescribeConfigs | 4 | 1 | -| 33 | AlterConfigs | 2 | 2 | -| 36 | SaslAuthenticate | 2 | 1 | -| 37 | CreatePartitions | 3 | 0 | -| 42 | DeleteGroups | 2 | 1 | -| 43 | ElectLeaders | 2 | 2 | -| 44 | IncrementalAlterConfigs | 1 | 1 | -| 47 | OffsetDelete | 0 | 0 | -| 50 | DescribeUserScramCredentials | 0 | 0 | -| 51 | AlterUserScramCredentials | 0 | 0 | -| 68 | ConsumerGroupHeartbeat | 1 | 1 | -| 69 | ConsumerGroupDescribe | 1 | 0 | -| 71 | GetTelemetrySubscriptions | 0 | 0 | -| 72 | PushTelemetry | 0 | 0 | +| ApiKey | Request name | Kafka max | librdkafka max | +| ------ | ---------------------------- | --------- | -------------- | +| 0 | Produce | 12 | 10 | +| 1 | Fetch | 17 | 16 | +| 2 | ListOffsets | 10 | 7 | +| 3 | Metadata | 13 | 13 | +| 8 | OffsetCommit | 9 | 9 | +| 9 | OffsetFetch | 9 | 9 | +| 10 | FindCoordinator | 6 | 2 | +| 11 | JoinGroup | 9 | 5 | +| 12 | Heartbeat | 4 | 3 | +| 13 | LeaveGroup | 5 | 1 | +| 14 | SyncGroup | 5 | 3 | +| 15 | DescribeGroups | 6 | 5 | +| 16 | ListGroups | 5 | 4 | +| 17 | SaslHandshake | 1 | 1 | +| 18 | ApiVersions | 4 | 3 | +| 19 | CreateTopics | 7 | 4 | +| 20 | DeleteTopics | 6 | 4 | +| 21 | DeleteRecords | 2 | 2 | +| 22 | InitProducerId | 5 | 4 | +| 23 | OffsetForLeaderEpoch | 4 | 2 | +| 24 | AddPartitionsToTxn | 5 | 0 | +| 25 | AddOffsetsToTxn | 4 | 0 | +| 26 | EndTxn | 5 | 1 | +| 28 | TxnOffsetCommit | 5 | 3 | +| 29 | DescribeAcls | 3 | 2 | +| 30 | CreateAcls | 3 | 2 | +| 31 | DeleteAcls | 3 | 2 | +| 32 | DescribeConfigs | 4 | 1 | +| 33 | AlterConfigs | 2 | 2 | +| 36 | SaslAuthenticate | 2 | 1 | +| 37 | CreatePartitions | 3 | 2 | +| 42 | DeleteGroups | 2 | 2 | +| 43 | ElectLeaders | 2 | 2 | +| 44 | IncrementalAlterConfigs | 1 | 1 | +| 47 | OffsetDelete | 0 | 0 | +| 50 | DescribeUserScramCredentials | 0 | 0 | +| 51 | AlterUserScramCredentials | 0 | 0 | +| 68 | ConsumerGroupHeartbeat | 1 | 1 | +| 69 | ConsumerGroupDescribe | 1 | 0 | +| 71 | GetTelemetrySubscriptions | 0 | 0 | +| 72 | PushTelemetry | 0 | 0 | # Recommendations for language binding developers diff --git a/lib/librdkafka-2.10.1/LICENSE b/lib/librdkafka-2.15.0/LICENSE similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE rename to lib/librdkafka-2.15.0/LICENSE diff --git a/lib/librdkafka-2.10.1/LICENSE.cjson b/lib/librdkafka-2.15.0/LICENSE.cjson similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.cjson rename to lib/librdkafka-2.15.0/LICENSE.cjson diff --git a/lib/librdkafka-2.10.1/LICENSE.crc32c b/lib/librdkafka-2.15.0/LICENSE.crc32c similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.crc32c rename to lib/librdkafka-2.15.0/LICENSE.crc32c diff --git a/lib/librdkafka-2.10.1/LICENSE.fnv1a b/lib/librdkafka-2.15.0/LICENSE.fnv1a similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.fnv1a rename to lib/librdkafka-2.15.0/LICENSE.fnv1a diff --git a/lib/librdkafka-2.10.1/LICENSE.hdrhistogram b/lib/librdkafka-2.15.0/LICENSE.hdrhistogram similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.hdrhistogram rename to lib/librdkafka-2.15.0/LICENSE.hdrhistogram diff --git a/lib/librdkafka-2.10.1/LICENSE.lz4 b/lib/librdkafka-2.15.0/LICENSE.lz4 similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.lz4 rename to lib/librdkafka-2.15.0/LICENSE.lz4 diff --git a/lib/librdkafka-2.10.1/LICENSE.murmur2 b/lib/librdkafka-2.15.0/LICENSE.murmur2 similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.murmur2 rename to lib/librdkafka-2.15.0/LICENSE.murmur2 diff --git a/lib/librdkafka-2.10.1/LICENSE.nanopb b/lib/librdkafka-2.15.0/LICENSE.nanopb similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.nanopb rename to lib/librdkafka-2.15.0/LICENSE.nanopb diff --git a/lib/librdkafka-2.10.1/LICENSE.opentelemetry b/lib/librdkafka-2.15.0/LICENSE.opentelemetry similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.opentelemetry rename to lib/librdkafka-2.15.0/LICENSE.opentelemetry diff --git a/lib/librdkafka-2.10.1/LICENSE.pycrc b/lib/librdkafka-2.15.0/LICENSE.pycrc similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.pycrc rename to lib/librdkafka-2.15.0/LICENSE.pycrc diff --git a/lib/librdkafka-2.10.1/LICENSE.queue b/lib/librdkafka-2.15.0/LICENSE.queue similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.queue rename to lib/librdkafka-2.15.0/LICENSE.queue diff --git a/lib/librdkafka-2.10.1/LICENSE.regexp b/lib/librdkafka-2.15.0/LICENSE.regexp similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.regexp rename to lib/librdkafka-2.15.0/LICENSE.regexp diff --git a/lib/librdkafka-2.10.1/LICENSE.snappy b/lib/librdkafka-2.15.0/LICENSE.snappy similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.snappy rename to lib/librdkafka-2.15.0/LICENSE.snappy diff --git a/lib/librdkafka-2.10.1/LICENSE.tinycthread b/lib/librdkafka-2.15.0/LICENSE.tinycthread similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.tinycthread rename to lib/librdkafka-2.15.0/LICENSE.tinycthread diff --git a/lib/librdkafka-2.10.1/LICENSE.wingetopt b/lib/librdkafka-2.15.0/LICENSE.wingetopt similarity index 100% rename from lib/librdkafka-2.10.1/LICENSE.wingetopt rename to lib/librdkafka-2.15.0/LICENSE.wingetopt diff --git a/lib/librdkafka-2.10.1/LICENSES.txt b/lib/librdkafka-2.15.0/LICENSES.txt similarity index 100% rename from lib/librdkafka-2.10.1/LICENSES.txt rename to lib/librdkafka-2.15.0/LICENSES.txt diff --git a/lib/librdkafka-2.10.1/Makefile b/lib/librdkafka-2.15.0/Makefile similarity index 97% rename from lib/librdkafka-2.10.1/Makefile rename to lib/librdkafka-2.15.0/Makefile index 6df5a3ad880..0d33e9e3439 100755 --- a/lib/librdkafka-2.10.1/Makefile +++ b/lib/librdkafka-2.15.0/Makefile @@ -9,7 +9,7 @@ DOC_FILES+= LICENSE LICENSES.txt INTRODUCTION.md README.md \ PKGNAME?= librdkafka VERSION?= $(shell python3 packaging/get_version.py src/rdkafka.h) -SHELL = bash +SHELL = /bin/sh # Jenkins CI integration BUILD_NUMBER ?= 1 @@ -80,7 +80,7 @@ rpm: distclean $(MAKE) -C packaging/rpm LICENSES.txt: .PHONY - @(for i in LICENSE LICENSE.*[^~] ; do (echo "$$i" ; echo "--------------------------------------------------------------" ; cat $$i ; echo "" ; echo "") ; done) > $@.tmp + @(for i in LICENSE LICENSE.*[!~] ; do (echo "$$i" ; echo "--------------------------------------------------------------" ; cat $$i ; echo "" ; echo "") ; done) > $@.tmp @cmp $@ $@.tmp || mv -f $@.tmp $@ ; rm -f $@.tmp diff --git a/lib/librdkafka-2.10.1/README.md b/lib/librdkafka-2.15.0/README.md similarity index 95% rename from lib/librdkafka-2.10.1/README.md rename to lib/librdkafka-2.15.0/README.md index 06f196bc0ee..82055eda0f7 100644 --- a/lib/librdkafka-2.10.1/README.md +++ b/lib/librdkafka-2.15.0/README.md @@ -1,5 +1,5 @@ -librdkafka - the Apache Kafka C/C++ client library -================================================== +librdkafka - the Apache Kafka® C/C++ client library +=================================================== Copyright (c) 2012-2022, [Magnus Edenhill](http://www.edenhill.se/). 2023 [Confluent Inc.](https://www.confluent.io/). @@ -19,10 +19,11 @@ has been licensed for use by librdkafka. librdkafka has no affiliation with and is not endorsed by The Apache Software Foundation. -# Features # +# Features * Full Exactly-Once-Semantics (EOS) support * High-level producer, including Idempotent and Transactional producers * High-level balanced KafkaConsumer (requires broker >= 0.9) + * Share consumer / Queues for Kafka ([KIP-932](INTRODUCTION.md#share-consumers-queues-for-kafka)) — **Preview**, C API only (requires broker >= 4.2) * Simple (legacy) consumer * Admin client * Compression: snappy, gzip, lz4, zstd @@ -147,6 +148,7 @@ See [getting Started with Apache Kafka and C/C++](https://developer.confluent.io * Producers: basic producers, idempotent producers, transactional producers. * Consumers: basic consumers, reading batches of messages. + * Share consumers (Queues for Kafka, KIP-932, Preview): see the `share_consumer*` examples. * Performance and latency testing tools. 2. Refer to the [examples GitHub repo](https://github.com/confluentinc/examples/tree/master/clients/cloud/c) for code connecting to a cloud streaming data service based on Apache Kafka @@ -168,7 +170,7 @@ File bug reports and feature requests using [GitHub Issues](https://github.com/c Questions and discussions are welcome on the [Discussions](https://github.com/confluentinc/librdkafka/discussions) forum, and on the [Confluent Community slack](https://launchpass.com/confluentcommunity) #clients channel. -# Language bindings # +# Language bindings * C#/.NET: [confluent-kafka-dotnet](https://github.com/confluentinc/confluent-kafka-dotnet) (based on [rdkafka-dotnet](https://github.com/ah-/rdkafka-dotnet)) * C++: [cppkafka](https://github.com/mfontanini/cppkafka) diff --git a/lib/librdkafka-2.10.1/README.win32 b/lib/librdkafka-2.15.0/README.win32 similarity index 100% rename from lib/librdkafka-2.10.1/README.win32 rename to lib/librdkafka-2.15.0/README.win32 diff --git a/lib/librdkafka-2.10.1/STATISTICS.md b/lib/librdkafka-2.15.0/STATISTICS.md similarity index 100% rename from lib/librdkafka-2.10.1/STATISTICS.md rename to lib/librdkafka-2.15.0/STATISTICS.md diff --git a/lib/librdkafka-2.10.1/configure b/lib/librdkafka-2.15.0/configure similarity index 100% rename from lib/librdkafka-2.10.1/configure rename to lib/librdkafka-2.15.0/configure diff --git a/lib/librdkafka-2.10.1/configure.self b/lib/librdkafka-2.15.0/configure.self similarity index 97% rename from lib/librdkafka-2.10.1/configure.self rename to lib/librdkafka-2.15.0/configure.self index 691278348aa..13a2c2848cb 100644 --- a/lib/librdkafka-2.10.1/configure.self +++ b/lib/librdkafka-2.15.0/configure.self @@ -216,6 +216,16 @@ void foo (void) { (void)rand_r(&seed); }" + # Check if getentropy() is available + mkl_compile_check "getentropy" "HAVE_GETENTROPY" disable CC "" \ +"#define _DEFAULT_SOURCE +#include +#include +int foo (void) { + char seed[16]; + return getentropy((void *)seed, sizeof(seed)); +}" + # Check if strndup() is available (isn't on Solaris 10) mkl_compile_check "strndup" "HAVE_STRNDUP" disable CC "" \ "#include diff --git a/lib/librdkafka-2.10.1/debian/.gitignore b/lib/librdkafka-2.15.0/debian/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/debian/.gitignore rename to lib/librdkafka-2.15.0/debian/.gitignore diff --git a/lib/librdkafka-2.10.1/debian/changelog b/lib/librdkafka-2.15.0/debian/changelog similarity index 100% rename from lib/librdkafka-2.10.1/debian/changelog rename to lib/librdkafka-2.15.0/debian/changelog diff --git a/lib/librdkafka-2.10.1/debian/compat b/lib/librdkafka-2.15.0/debian/compat similarity index 100% rename from lib/librdkafka-2.10.1/debian/compat rename to lib/librdkafka-2.15.0/debian/compat diff --git a/lib/librdkafka-2.10.1/debian/control b/lib/librdkafka-2.15.0/debian/control similarity index 100% rename from lib/librdkafka-2.10.1/debian/control rename to lib/librdkafka-2.15.0/debian/control diff --git a/lib/librdkafka-2.10.1/debian/copyright b/lib/librdkafka-2.15.0/debian/copyright similarity index 100% rename from lib/librdkafka-2.10.1/debian/copyright rename to lib/librdkafka-2.15.0/debian/copyright diff --git a/lib/librdkafka-2.10.1/debian/gbp.conf b/lib/librdkafka-2.15.0/debian/gbp.conf similarity index 100% rename from lib/librdkafka-2.10.1/debian/gbp.conf rename to lib/librdkafka-2.15.0/debian/gbp.conf diff --git a/lib/librdkafka-2.10.1/debian/librdkafka++1.install b/lib/librdkafka-2.15.0/debian/librdkafka++1.install similarity index 100% rename from lib/librdkafka-2.10.1/debian/librdkafka++1.install rename to lib/librdkafka-2.15.0/debian/librdkafka++1.install diff --git a/lib/librdkafka-2.10.1/debian/librdkafka-dev.examples b/lib/librdkafka-2.15.0/debian/librdkafka-dev.examples similarity index 100% rename from lib/librdkafka-2.10.1/debian/librdkafka-dev.examples rename to lib/librdkafka-2.15.0/debian/librdkafka-dev.examples diff --git a/lib/librdkafka-2.10.1/debian/librdkafka-dev.install b/lib/librdkafka-2.15.0/debian/librdkafka-dev.install similarity index 100% rename from lib/librdkafka-2.10.1/debian/librdkafka-dev.install rename to lib/librdkafka-2.15.0/debian/librdkafka-dev.install diff --git a/lib/librdkafka-2.10.1/debian/librdkafka1.docs b/lib/librdkafka-2.15.0/debian/librdkafka1.docs similarity index 100% rename from lib/librdkafka-2.10.1/debian/librdkafka1.docs rename to lib/librdkafka-2.15.0/debian/librdkafka1.docs diff --git a/lib/librdkafka-2.10.1/debian/librdkafka1.install b/lib/librdkafka-2.15.0/debian/librdkafka1.install similarity index 100% rename from lib/librdkafka-2.10.1/debian/librdkafka1.install rename to lib/librdkafka-2.15.0/debian/librdkafka1.install diff --git a/lib/librdkafka-2.10.1/debian/librdkafka1.symbols b/lib/librdkafka-2.15.0/debian/librdkafka1.symbols similarity index 100% rename from lib/librdkafka-2.10.1/debian/librdkafka1.symbols rename to lib/librdkafka-2.15.0/debian/librdkafka1.symbols diff --git a/lib/librdkafka-2.10.1/debian/rules b/lib/librdkafka-2.15.0/debian/rules similarity index 100% rename from lib/librdkafka-2.10.1/debian/rules rename to lib/librdkafka-2.15.0/debian/rules diff --git a/lib/librdkafka-2.10.1/debian/source/format b/lib/librdkafka-2.15.0/debian/source/format similarity index 100% rename from lib/librdkafka-2.10.1/debian/source/format rename to lib/librdkafka-2.15.0/debian/source/format diff --git a/lib/librdkafka-2.10.1/debian/watch b/lib/librdkafka-2.15.0/debian/watch similarity index 100% rename from lib/librdkafka-2.10.1/debian/watch rename to lib/librdkafka-2.15.0/debian/watch diff --git a/lib/librdkafka-2.10.1/dev-conf.sh b/lib/librdkafka-2.15.0/dev-conf.sh similarity index 100% rename from lib/librdkafka-2.10.1/dev-conf.sh rename to lib/librdkafka-2.15.0/dev-conf.sh diff --git a/lib/librdkafka-2.10.1/examples/.gitignore b/lib/librdkafka-2.15.0/examples/.gitignore similarity index 84% rename from lib/librdkafka-2.10.1/examples/.gitignore rename to lib/librdkafka-2.15.0/examples/.gitignore index a2cd3d609a4..8e0e7a7874e 100644 --- a/lib/librdkafka-2.10.1/examples/.gitignore +++ b/lib/librdkafka-2.15.0/examples/.gitignore @@ -22,4 +22,7 @@ alter_consumer_group_offsets incremental_alter_configs user_scram list_offsets -elect_leaders \ No newline at end of file +elect_leaders +share_consumer +share_consumer_commit_async +share_consumer_commit_sync diff --git a/lib/librdkafka-2.10.1/examples/CMakeLists.txt b/lib/librdkafka-2.15.0/examples/CMakeLists.txt similarity index 89% rename from lib/librdkafka-2.10.1/examples/CMakeLists.txt rename to lib/librdkafka-2.15.0/examples/CMakeLists.txt index 91851d2cbf2..d9ce7ac277f 100644 --- a/lib/librdkafka-2.10.1/examples/CMakeLists.txt +++ b/lib/librdkafka-2.15.0/examples/CMakeLists.txt @@ -76,4 +76,13 @@ if(NOT WIN32) add_executable(kafkatest_verifiable_client kafkatest_verifiable_client.cpp) target_link_libraries(kafkatest_verifiable_client PUBLIC rdkafka++) + add_executable(share_consumer share_consumer.c) + target_link_libraries(share_consumer PUBLIC rdkafka) + + add_executable(share_consumer_commit_sync share_consumer_commit_sync.c) + target_link_libraries(share_consumer_commit_sync PUBLIC rdkafka) + + add_executable(share_consumer_commit_async share_consumer_commit_async.c) + target_link_libraries(share_consumer_commit_async PUBLIC rdkafka) + endif(NOT WIN32) diff --git a/lib/librdkafka-2.10.1/examples/Makefile b/lib/librdkafka-2.15.0/examples/Makefile similarity index 90% rename from lib/librdkafka-2.10.1/examples/Makefile rename to lib/librdkafka-2.15.0/examples/Makefile index f8b6513540e..ee6e5c86724 100644 --- a/lib/librdkafka-2.10.1/examples/Makefile +++ b/lib/librdkafka-2.15.0/examples/Makefile @@ -1,7 +1,9 @@ EXAMPLES ?= rdkafka_example rdkafka_performance rdkafka_example_cpp \ rdkafka_complex_consumer_example rdkafka_complex_consumer_example_cpp \ kafkatest_verifiable_client \ - producer consumer idempotent_producer transactions \ + producer consumer share_consumer share_consumer_commit_async \ + share_consumer_commit_sync \ + idempotent_producer transactions \ delete_records \ openssl_engine_example_cpp \ list_consumer_groups \ @@ -54,6 +56,18 @@ consumer: ../src/librdkafka.a consumer.c $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ ../src/librdkafka.a $(LIBS) +share_consumer: ../src/librdkafka.a share_consumer.c + $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ + ../src/librdkafka.a $(LIBS) + +share_consumer_commit_async: ../src/librdkafka.a share_consumer_commit_async.c + $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ + ../src/librdkafka.a $(LIBS) + +share_consumer_commit_sync: ../src/librdkafka.a share_consumer_commit_sync.c + $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ + ../src/librdkafka.a $(LIBS) + idempotent_producer: ../src/librdkafka.a idempotent_producer.c $(CC) $(CPPFLAGS) $(CFLAGS) $@.c -o $@ $(LDFLAGS) \ ../src/librdkafka.a $(LIBS) diff --git a/lib/librdkafka-2.10.1/examples/README.md b/lib/librdkafka-2.15.0/examples/README.md similarity index 85% rename from lib/librdkafka-2.10.1/examples/README.md rename to lib/librdkafka-2.15.0/examples/README.md index 236291c5496..9aebf938463 100644 --- a/lib/librdkafka-2.10.1/examples/README.md +++ b/lib/librdkafka-2.15.0/examples/README.md @@ -19,6 +19,12 @@ Begin with the following examples: * [misc.c](misc.c) - a collection of miscellaneous usage examples. +For the share consumer (Queues for Kafka, [KIP-932](../INTRODUCTION.md#share-consumers-queues-for-kafka), **Preview**), see: + * [share_consumer.c](share_consumer.c) - a basic share consumer (poll + acknowledge). + * [share_consumer_commit_sync.c](share_consumer_commit_sync.c) - share consumer with explicit acknowledgement and synchronous commit. + * [share_consumer_commit_async.c](share_consumer_commit_async.c) - share consumer with explicit acknowledgement and asynchronous commit. + + For more complex uses, see: * [rdkafka_example.c](rdkafka_example.c) - simple consumer, producer, metadata listing, kitchen sink, etc. * [rdkafka_example.cpp](rdkafka_example.cpp) - simple consumer, producer, metadata listing in C++. diff --git a/lib/librdkafka-2.10.1/examples/alter_consumer_group_offsets.c b/lib/librdkafka-2.15.0/examples/alter_consumer_group_offsets.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/alter_consumer_group_offsets.c rename to lib/librdkafka-2.15.0/examples/alter_consumer_group_offsets.c diff --git a/lib/librdkafka-2.10.1/examples/consumer.c b/lib/librdkafka-2.15.0/examples/consumer.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/consumer.c rename to lib/librdkafka-2.15.0/examples/consumer.c diff --git a/lib/librdkafka-2.10.1/examples/delete_records.c b/lib/librdkafka-2.15.0/examples/delete_records.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/delete_records.c rename to lib/librdkafka-2.15.0/examples/delete_records.c diff --git a/lib/librdkafka-2.10.1/examples/describe_cluster.c b/lib/librdkafka-2.15.0/examples/describe_cluster.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/describe_cluster.c rename to lib/librdkafka-2.15.0/examples/describe_cluster.c diff --git a/lib/librdkafka-2.10.1/examples/describe_consumer_groups.c b/lib/librdkafka-2.15.0/examples/describe_consumer_groups.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/describe_consumer_groups.c rename to lib/librdkafka-2.15.0/examples/describe_consumer_groups.c diff --git a/lib/librdkafka-2.10.1/examples/describe_topics.c b/lib/librdkafka-2.15.0/examples/describe_topics.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/describe_topics.c rename to lib/librdkafka-2.15.0/examples/describe_topics.c diff --git a/lib/librdkafka-2.10.1/examples/elect_leaders.c b/lib/librdkafka-2.15.0/examples/elect_leaders.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/elect_leaders.c rename to lib/librdkafka-2.15.0/examples/elect_leaders.c diff --git a/lib/librdkafka-2.10.1/examples/globals.json b/lib/librdkafka-2.15.0/examples/globals.json similarity index 100% rename from lib/librdkafka-2.10.1/examples/globals.json rename to lib/librdkafka-2.15.0/examples/globals.json diff --git a/lib/librdkafka-2.10.1/examples/idempotent_producer.c b/lib/librdkafka-2.15.0/examples/idempotent_producer.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/idempotent_producer.c rename to lib/librdkafka-2.15.0/examples/idempotent_producer.c diff --git a/lib/librdkafka-2.10.1/examples/incremental_alter_configs.c b/lib/librdkafka-2.15.0/examples/incremental_alter_configs.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/incremental_alter_configs.c rename to lib/librdkafka-2.15.0/examples/incremental_alter_configs.c diff --git a/lib/librdkafka-2.10.1/examples/kafkatest_verifiable_client.cpp b/lib/librdkafka-2.15.0/examples/kafkatest_verifiable_client.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/kafkatest_verifiable_client.cpp rename to lib/librdkafka-2.15.0/examples/kafkatest_verifiable_client.cpp diff --git a/lib/librdkafka-2.10.1/examples/list_consumer_group_offsets.c b/lib/librdkafka-2.15.0/examples/list_consumer_group_offsets.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/list_consumer_group_offsets.c rename to lib/librdkafka-2.15.0/examples/list_consumer_group_offsets.c diff --git a/lib/librdkafka-2.10.1/examples/list_consumer_groups.c b/lib/librdkafka-2.15.0/examples/list_consumer_groups.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/list_consumer_groups.c rename to lib/librdkafka-2.15.0/examples/list_consumer_groups.c diff --git a/lib/librdkafka-2.10.1/examples/list_offsets.c b/lib/librdkafka-2.15.0/examples/list_offsets.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/list_offsets.c rename to lib/librdkafka-2.15.0/examples/list_offsets.c diff --git a/lib/librdkafka-2.10.1/examples/misc.c b/lib/librdkafka-2.15.0/examples/misc.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/misc.c rename to lib/librdkafka-2.15.0/examples/misc.c diff --git a/lib/librdkafka-2.10.1/examples/openssl_engine_example.cpp b/lib/librdkafka-2.15.0/examples/openssl_engine_example.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/openssl_engine_example.cpp rename to lib/librdkafka-2.15.0/examples/openssl_engine_example.cpp diff --git a/lib/librdkafka-2.10.1/examples/producer.c b/lib/librdkafka-2.15.0/examples/producer.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/producer.c rename to lib/librdkafka-2.15.0/examples/producer.c diff --git a/lib/librdkafka-2.10.1/examples/producer.cpp b/lib/librdkafka-2.15.0/examples/producer.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/producer.cpp rename to lib/librdkafka-2.15.0/examples/producer.cpp diff --git a/lib/librdkafka-2.10.1/examples/rdkafka_complex_consumer_example.c b/lib/librdkafka-2.15.0/examples/rdkafka_complex_consumer_example.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/rdkafka_complex_consumer_example.c rename to lib/librdkafka-2.15.0/examples/rdkafka_complex_consumer_example.c diff --git a/lib/librdkafka-2.10.1/examples/rdkafka_complex_consumer_example.cpp b/lib/librdkafka-2.15.0/examples/rdkafka_complex_consumer_example.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/rdkafka_complex_consumer_example.cpp rename to lib/librdkafka-2.15.0/examples/rdkafka_complex_consumer_example.cpp diff --git a/lib/librdkafka-2.10.1/examples/rdkafka_consume_batch.cpp b/lib/librdkafka-2.15.0/examples/rdkafka_consume_batch.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/rdkafka_consume_batch.cpp rename to lib/librdkafka-2.15.0/examples/rdkafka_consume_batch.cpp diff --git a/lib/librdkafka-2.10.1/examples/rdkafka_example.c b/lib/librdkafka-2.15.0/examples/rdkafka_example.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/rdkafka_example.c rename to lib/librdkafka-2.15.0/examples/rdkafka_example.c diff --git a/lib/librdkafka-2.10.1/examples/rdkafka_example.cpp b/lib/librdkafka-2.15.0/examples/rdkafka_example.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/rdkafka_example.cpp rename to lib/librdkafka-2.15.0/examples/rdkafka_example.cpp diff --git a/lib/librdkafka-2.10.1/examples/rdkafka_performance.c b/lib/librdkafka-2.15.0/examples/rdkafka_performance.c similarity index 93% rename from lib/librdkafka-2.10.1/examples/rdkafka_performance.c rename to lib/librdkafka-2.15.0/examples/rdkafka_performance.c index dab0b06b8f6..0255ad5c54b 100644 --- a/lib/librdkafka-2.10.1/examples/rdkafka_performance.c +++ b/lib/librdkafka-2.15.0/examples/rdkafka_performance.c @@ -705,7 +705,8 @@ print_stats(rd_kafka_t *rk, int mode, int otype, const char *compression) { static void sig_usr1(int sig) { - rd_kafka_dump(stdout, global_rk); + if (global_rk) + rd_kafka_dump(stdout, global_rk); } @@ -878,24 +879,13 @@ int main(int argc, char **argv) { rd_kafka_conf_set(conf, "message.send.max.retries", "3", NULL, 0); rd_kafka_conf_set(conf, "retry.backoff.ms", "500", NULL, 0); - /* Consumer config */ - /* Tell rdkafka to (try to) maintain 1M messages - * in its internal receive buffers. This is to avoid - * application -> rdkafka -> broker per-message ping-pong - * latency. - * The larger the local queue, the higher the performance. - * Try other values with: ... -X queued.min.messages=1000 - */ - rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, 0); - rd_kafka_conf_set(conf, "session.timeout.ms", "6000", NULL, 0); - rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", NULL, 0); - topics = rd_kafka_topic_partition_list_new(1); while ((opt = getopt(argc, argv, - "PCG:t:p:b:s:k:c:fi:MDd:m:S:x:" + "PCG:S:t:p:b:s:k:c:fi:MDd:m:F:x:" "R:a:z:o:X:B:eT:Y:qvIur:lA:OwNH:")) != -1) { switch (opt) { + case 'S': case 'G': if (rd_kafka_conf_set(conf, "group.id", optarg, errstr, sizeof(errstr)) != @@ -940,7 +930,7 @@ int main(int argc, char **argv) { case 'm': msgpattern = optarg; break; - case 'S': + case 'F': seq = strtoull(optarg, NULL, 10); do_seq = 1; break; @@ -1138,6 +1128,7 @@ int main(int argc, char **argv) { " Options:\n" " -C | -P | Consumer or Producer mode\n" " -G High-level Kafka Consumer mode\n" + " -S Share Consumer mode\n" " -t Topic to consume / produce\n" " -p Partition (defaults to random). " "Multiple partitions are allowed in -C consumer mode.\n" @@ -1153,7 +1144,7 @@ int main(int argc, char **argv) { " -D Copy/Duplicate data buffer (producer)\n" " -i Display interval\n" " -m Message payload pattern\n" - " -S Send a sequence number starting at " + " -F Send a sequence number starting at " " as payload\n" " -R Random seed value (defaults to time)\n" " -a Required acks (producer): " @@ -1195,12 +1186,30 @@ int main(int argc, char **argv) { "\n" " In Producer mode:\n" " writes messages of size -s <..> and prints thruput\n" + "\n" + " In Share Consumer mode (-S):\n" + " consumes messages using share groups and prints " + "thruput\n" "\n", argv[0], rd_kafka_version_str(), rd_kafka_version(), RD_KAFKA_DEBUG_CONTEXTS); exit(1); } + if (mode == 'C' || mode == 'G') { + /* Tell rdkafka to (try to) maintain 1M messages + * in its internal receive buffers. This is to avoid + * application -> rdkafka -> broker per-message ping-pong + * latency. + * The larger the local queue, the higher the performance. + * Try other values with: ... -X queued.min.messages=1000 + */ + rd_kafka_conf_set(conf, "queued.min.messages", "1000000", NULL, + 0); + rd_kafka_conf_set(conf, "session.timeout.ms", "6000", NULL, 0); + rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", NULL, + 0); + } dispintvl *= 1000; /* us */ @@ -1314,6 +1323,33 @@ int main(int argc, char **argv) { rd_kafka_conf_set(conf, "enable.partition.eof", "true", NULL, 0); + if (mode == 'S') { + if (partition_cnt > 0) { + fprintf(stderr, + "%% -p is not supported with " + "share consumer mode\n"); + exit(1); + } + if (start_offset != 0) { + fprintf(stderr, + "%% -o is not supported with " + "share consumer mode\n"); + exit(1); + } + if (exit_eof) { + fprintf(stderr, + "%% -e is not supported with " + "share consumer mode\n"); + exit(1); + } + if (batch_size) { + fprintf(stderr, + "%% -B is not supported with " + "share consumer mode\n"); + exit(1); + } + } + if (read_hdrs && mode == 'P') { fprintf(stderr, "%% producer can not read headers\n"); exit(1); @@ -1748,6 +1784,68 @@ int main(int argc, char **argv) { free(rkmessages); rd_kafka_queue_destroy(rkqu); rd_kafka_destroy(rk); + + } else if (mode == 'S') { + /* Share Consumer */ + rd_kafka_share_t *rkshare; + + /* Create share consumer instance. */ + rkshare = + rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + if (!rkshare) { + fprintf(stderr, + "%% Failed to create share consumer: %s\n", + errstr); + exit(1); + } + + err = rd_kafka_share_subscribe(rkshare, topics); + if (err) { + fprintf(stderr, "%% Share subscribe failed: %s\n", + rd_kafka_err2str(err)); + exit(1); + } + fprintf(stderr, + "%% Share consumer subscribed, " + "waiting for messages...\n"); + + while (run && (msgcnt == -1 || msgcnt > (int)cnt.msgs)) { + uint64_t fetch_latency; + rd_kafka_messages_t *rkmessages = NULL; + size_t rcvd_msgs; + size_t i; + rd_kafka_error_t *error; + + fetch_latency = rd_clock(); + + error = rd_kafka_share_poll(rkshare, 1000, &rkmessages); + + cnt.t_fetch_latency += rd_clock() - fetch_latency; + + if (error) { + fprintf(stderr, "%% Share consume error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(rkmessages); + continue; + } + + rcvd_msgs = rd_kafka_messages_count(rkmessages); + for (i = 0; i < rcvd_msgs; i++) + msg_consume( + rd_kafka_messages_get(rkmessages, i), NULL); + rd_kafka_messages_destroy(rkmessages); + + if (rcvd_msgs > 0 && rate_sleep) + do_sleep(rate_sleep); + + print_stats(NULL, mode, otype, compression); + } + cnt.t_end = rd_clock(); + + fprintf(stderr, "%% Closing share consumer\n"); + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); } if (hdrs) diff --git a/lib/librdkafka-2.15.0/examples/share_consumer.c b/lib/librdkafka-2.15.0/examples/share_consumer.c new file mode 100644 index 00000000000..8323d4da8b5 --- /dev/null +++ b/lib/librdkafka-2.15.0/examples/share_consumer.c @@ -0,0 +1,280 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2019-2022, Magnus Edenhill + * 2023, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Example KIP-932 share consumer in the default (implicit) ack mode. + * + * Consumers in a share group share partitions like a queue. In implicit + * mode each record is acknowledged for you on the next rd_kafka_share_poll(); + * there is nothing to ack from the application. If this consumer crashes + * before the next poll, the records it held are redelivered to another + * consumer in the group. + * + * Usage: + * share_consumer [topic2 ...] + */ + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include +#include +#include +#include + + +/* Typical include path would be , but this program + * is builtin from within the librdkafka source tree and thus differs. */ +// #include +#include "rdkafka.h" + + +static volatile sig_atomic_t run = 1; + +/** + * @brief Signal termination of program + */ +static void stop(int sig) { + run = 0; +} + + + +/** + * @returns 1 if all bytes are printable, else 0. + */ +static int is_printable(const char *buf, size_t size) { + size_t i; + + for (i = 0; i < size; i++) + if (!isprint((int)buf[i])) + return 0; + + return 1; +} + + +int main(int argc, char **argv) { + rd_kafka_share_t *rkshare; /* Consumer instance handle */ + rd_kafka_conf_t *conf; /* Temporary configuration object */ + rd_kafka_resp_err_t err; /* librdkafka API error code */ + char errstr[512]; /* librdkafka API error reporting buffer */ + const char *brokers; /* Argument: broker list */ + const char *groupid; /* Argument: Consumer group id */ + char **topics; /* Argument: list of topics to subscribe to */ + int topic_cnt; /* Number of topics to subscribe to */ + rd_kafka_topic_partition_list_t *subscription; /* Subscribed topics */ + int i; + int ret = 0; /* Process exit code */ + + /* + * Argument validation + */ + if (argc < 3) { + fprintf(stderr, + "%% Usage: " + "%s " + "..\n", + argv[0]); + return 1; + } + + brokers = argv[1]; + groupid = argv[2]; + topics = &argv[3]; + topic_cnt = argc - 3; + + + /* + * Create Kafka client configuration place-holder + */ + conf = rd_kafka_conf_new(); + + /* Set bootstrap broker(s) as a comma-separated list of + * host or host:port (default port 9092). + * librdkafka will use the bootstrap brokers to acquire the full + * set of brokers from the cluster. */ + if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + /* Set the consumer group id. + * All consumers sharing the same group id will join the same + * group, and the subscribed topic' partitions will be assigned + * according to the partition.assignment.strategy + * (consumer config property) to the consumers in the group. */ + if (rd_kafka_conf_set(conf, "group.id", groupid, errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + /* + * Create a new share consumer instance. + * + * NOTE: rd_kafka_share_consumer_new() takes ownership of the conf + * object and the application must not reference it again after this + * call. + */ + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + if (!rkshare) { + fprintf(stderr, + "%% Failed to create new share consumer: " + "%s\n", + errstr); + return 1; + } + + conf = NULL; /* Configuration object is now owned, and freed, + * by the rd_kafka_t instance. */ + + + /* Convert the list of topics to a format suitable for librdkafka */ + subscription = rd_kafka_topic_partition_list_new(topic_cnt); + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(subscription, topics[i], + /* the partition is ignored + * by subscribe() */ + RD_KAFKA_PARTITION_UA); + + /* Subscribe to the list of topics */ + err = rd_kafka_share_subscribe(rkshare, subscription); + if (err) { + fprintf(stderr, + "%% Failed to subscribe to %d topics: " + "%s\n", + subscription->cnt, rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subscription); + rd_kafka_share_destroy(rkshare); + return 1; + } + + fprintf(stderr, + "%% Subscribed to %d topic(s), " + "waiting for rebalance and messages...\n", + subscription->cnt); + + rd_kafka_topic_partition_list_destroy(subscription); + + + /* Signal handler for clean shutdown */ + signal(SIGINT, stop); + + /* Subscribing to topics will trigger a group rebalance + * which may take some time to finish, but there is no need + * for the application to handle this idle period in a special way + * since a rebalance may happen at any time. + * Start polling for messages. */ + + while (run) { + rd_kafka_messages_t *rkmessages = NULL; + size_t rcvd_msgs = 0; + size_t i; + rd_kafka_error_t *error; + + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + int fatal = rd_kafka_error_is_fatal(error); + fprintf(stderr, "%% Consume error%s: %s\n", + fatal ? " (fatal)" : "", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(rkmessages); + /* A fatal error is unrecoverable: stop consuming. */ + if (fatal) { + ret = 1; + goto done; + } + continue; + } + + rcvd_msgs = rd_kafka_messages_count(rkmessages); + fprintf(stderr, "%% Received %zu messages\n", rcvd_msgs); + for (i = 0; i < rcvd_msgs; i++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(rkmessages, i); + + if (rkm->err) { + /* A record delivered with an error has already + * been acknowledged for you by the library: + * RELEASE (put back for retry) for + * decompression errors, and REJECT (give up) + * for CRC or unsupported-format errors. You can + * still override that by acknowledging the + * offset yourself if your application needs + * different handling. */ + fprintf(stderr, + "%% Consumer error: %d: " + "%s\n", + rkm->err, rd_kafka_message_errstr(rkm)); + continue; + } + + /* Proper message. */ + printf("Message received on %s [%" PRId32 + "] at offset %" PRId64, + rd_kafka_topic_name(rkm->rkt), rkm->partition, + rkm->offset); + + /* Print the message key. */ + if (rkm->key && is_printable(rkm->key, rkm->key_len)) + printf(" Key: %.*s\n", (int)rkm->key_len, + (const char *)rkm->key); + else if (rkm->key) + printf(" Key: (%d bytes)\n", (int)rkm->key_len); + + /* Print the message value/payload. */ + if (rkm->payload && + is_printable(rkm->payload, rkm->len)) + printf(" - Value: %.*s\n", (int)rkm->len, + (const char *)rkm->payload); + else if (rkm->payload) + printf(" - Value: (%d bytes)\n", (int)rkm->len); + } + rd_kafka_messages_destroy(rkmessages); + } + + +done: + /* Close the consumer: commit final offsets and leave the group. */ + fprintf(stderr, "%% Closing share consumer\n"); + rd_kafka_share_consumer_close(rkshare); + + + /* Destroy the consumer */ + rd_kafka_share_destroy(rkshare); + + return ret; +} diff --git a/lib/librdkafka-2.15.0/examples/share_consumer_commit_async.c b/lib/librdkafka-2.15.0/examples/share_consumer_commit_async.c new file mode 100644 index 00000000000..93ceeeabb09 --- /dev/null +++ b/lib/librdkafka-2.15.0/examples/share_consumer_commit_async.c @@ -0,0 +1,338 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Share consumer example using rd_kafka_share_commit_async() to + * explicitly acknowledge and commit records between polls. + * + * Usage: + * share_consumer_commit_async [topic2 ...] + * + * This example demonstrates: + * - Consuming records with rd_kafka_share_poll() + * - Explicitly acknowledging each record with + * rd_kafka_share_acknowledge_type() (ACCEPT) + * - Committing the batch's acknowledgements asynchronously with + * rd_kafka_share_commit_async() once per poll; the broker's reply is + * delivered later to the acknowledgement-commit callback + */ + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include +#include +#include +#include + +/* Typical include path would be , but this program + * is builtin from within the librdkafka source tree and thus differs. */ +#include "rdkafka.h" + + +static volatile sig_atomic_t run = 1; + +/** + * @brief Signal termination of program + */ +static void stop(int sig) { + run = 0; +} + + +/** + * @returns 1 if all bytes are printable, else 0. + */ +static int is_printable(const char *buf, size_t size) { + size_t i; + + for (i = 0; i < size; i++) + if (!isprint((int)buf[i])) + return 0; + + return 1; +} + + +/** + * @brief Acknowledgement-commit callback. + * + * rd_kafka_share_commit_async() does not block; its result is reported here. + * @p partitions lists each partition with the offsets that were committed; + * @p err is set if the commit failed. + */ +static void +acknowledgement_commit_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + size_t i, cnt; + + if (err) { + fprintf(stderr, "%% Acknowledgement commit failed: %s\n", + rd_kafka_err2str(err)); + return; + } + + cnt = rd_kafka_share_partition_offsets_list_count(partitions); + for (i = 0; i < cnt; i++) { + const rd_kafka_share_partition_offsets_t *po = + rd_kafka_share_partition_offsets_list_get(partitions, i); + const rd_kafka_topic_partition_t *tp = + rd_kafka_share_partition_offsets_partition(po); + size_t offset_cnt = + rd_kafka_share_partition_offsets_offsets_cnt(po); + + fprintf(stderr, + "%% Committed %s [%" PRId32 "]: %zu offset(s)\n", + tp->topic, tp->partition, offset_cnt); + } +} + + +/** + * @brief Process a received record. + * + * Replace this with your own handling. The return value tells the caller how + * to acknowledge the record: + * 0 success -> ACCEPT (done, never redeliver it) + * > 0 transient failure -> RELEASE (retry later, maybe another consumer) + * < 0 permanent failure -> REJECT (give up; the broker archives it) + */ +static int process_message(const rd_kafka_message_t *rkm) { + printf("Message on %s [%" PRId32 "] at offset %" PRId64, + rd_kafka_topic_name(rkm->rkt), rkm->partition, rkm->offset); + + if (rkm->key && is_printable(rkm->key, rkm->key_len)) + printf(" Key: %.*s", (int)rkm->key_len, (const char *)rkm->key); + + if (rkm->payload && is_printable(rkm->payload, rkm->len)) + printf(" Value: %.*s", (int)rkm->len, + (const char *)rkm->payload); + + printf("\n"); + + return 0; +} + + +int main(int argc, char **argv) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_kafka_resp_err_t err; + char errstr[512]; + const char *brokers; + const char *groupid; + char **topics; + int topic_cnt; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_error_t *cb_error; + int i; + int ret = 0; /* Process exit code */ + + if (argc < 4) { + fprintf(stderr, + "%% Usage: " + "%s [topic2 ..]\n", + argv[0]); + return 1; + } + + brokers = argv[1]; + groupid = argv[2]; + topics = &argv[3]; + topic_cnt = argc - 3; + + conf = rd_kafka_conf_new(); + + if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + if (rd_kafka_conf_set(conf, "group.id", groupid, errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + if (rd_kafka_conf_set(conf, "share.acknowledgement.mode", "explicit", + errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + if (!rkshare) { + fprintf(stderr, "%% Failed to create new share consumer: %s\n", + errstr); + return 1; + } + + conf = NULL; + + /* Register the acknowledgement-commit callback that + * rd_kafka_share_commit_async() reports its result to. */ + cb_error = rd_kafka_share_set_acknowledgement_commit_cb( + rkshare, acknowledgement_commit_cb, NULL); + if (cb_error) { + fprintf(stderr, + "%% Failed to set acknowledgement commit callback: " + "%s\n", + rd_kafka_error_string(cb_error)); + rd_kafka_error_destroy(cb_error); + rd_kafka_share_destroy(rkshare); + return 1; + } + + subscription = rd_kafka_topic_partition_list_new(topic_cnt); + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(subscription, topics[i], + RD_KAFKA_PARTITION_UA); + + err = rd_kafka_share_subscribe(rkshare, subscription); + if (err) { + fprintf(stderr, "%% Failed to subscribe to %d topics: %s\n", + subscription->cnt, rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subscription); + rd_kafka_share_destroy(rkshare); + return 1; + } + + fprintf(stderr, + "%% Subscribed to %d topic(s), " + "waiting for rebalance and messages...\n", + subscription->cnt); + + rd_kafka_topic_partition_list_destroy(subscription); + + signal(SIGINT, stop); + + while (run) { + rd_kafka_messages_t *rkmessages = NULL; + rd_kafka_error_t *error; + size_t rcvd_msgs; + size_t k; + + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + int fatal = rd_kafka_error_is_fatal(error); + fprintf(stderr, "%% Consume error%s: %s\n", + fatal ? " (fatal)" : "", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(rkmessages); + /* A fatal error is unrecoverable: stop consuming. */ + if (fatal) { + ret = 1; + goto done; + } + continue; + } + + rcvd_msgs = rd_kafka_messages_count(rkmessages); + if (rcvd_msgs == 0) { + rd_kafka_messages_destroy(rkmessages); + continue; + } + + printf("Received %zu messages\n", rcvd_msgs); + + for (k = 0; k < rcvd_msgs; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(rkmessages, k); + rd_kafka_share_AcknowledgeType_t ack_type; + int rc; + + if (rkm->err) { + /* A record delivered with an error has already + * been acknowledged for you by the library: + * RELEASE (put back for retry) for + * decompression errors, and REJECT (give up) + * for CRC or unsupported-format errors. You can + * still override that by acknowledging the + * offset yourself if your application needs + * different handling. */ + fprintf(stderr, "%% Consumer error: %d: %s\n", + rkm->err, rd_kafka_message_errstr(rkm)); + continue; + } + + /* Decide how to acknowledge based on the processing + * outcome: ACCEPT on success, RELEASE on a transient + * failure so the record can be retried, REJECT on a + * permanent failure so the broker archives it. */ + rc = process_message(rkm); + if (rc == 0) + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + else if (rc > 0) + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE; + else + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT; + + err = rd_kafka_share_acknowledge_type(rkshare, rkm, + ack_type); + if (err) + fprintf(stderr, + "%% Acknowledge error for " + "%s [%" PRId32 "] @ %" PRId64 ": %s\n", + rd_kafka_topic_name(rkm->rkt), + rkm->partition, rkm->offset, + rd_kafka_err2str(err)); + } + + /* Flush this batch's acknowledgements before the next poll(). + * commit_async() does not block; the result is delivered to + * the acknowledgement-commit callback. */ + error = rd_kafka_share_commit_async(rkshare); + if (error) { + fprintf(stderr, "%% Commit async error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } + + rd_kafka_messages_destroy(rkmessages); + } + +done: + fprintf(stderr, "%% Closing share consumer\n"); + rd_kafka_share_consumer_close(rkshare); + + rd_kafka_share_destroy(rkshare); + + return ret; +} diff --git a/lib/librdkafka-2.15.0/examples/share_consumer_commit_sync.c b/lib/librdkafka-2.15.0/examples/share_consumer_commit_sync.c new file mode 100644 index 00000000000..bc83fd6edeb --- /dev/null +++ b/lib/librdkafka-2.15.0/examples/share_consumer_commit_sync.c @@ -0,0 +1,304 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Share consumer example using rd_kafka_share_commit_sync() to + * explicitly acknowledge and synchronously commit records. + * + * Usage: + * share_consumer_commit_sync [topic2 ...] + * + * This example demonstrates: + * - Consuming records with rd_kafka_share_poll() + * - Explicitly acknowledging each record with + * rd_kafka_share_acknowledge_type(): ACCEPT on successful processing, + * RELEASE on a processing failure so the record can be redelivered + * - Committing the batch's acknowledgements synchronously with + * rd_kafka_share_commit_sync() once per poll, printing any + * per-partition errors + */ + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include +#include +#include +#include + +/* Typical include path would be , but this program + * is builtin from within the librdkafka source tree and thus differs. */ +#include "rdkafka.h" + + +static volatile sig_atomic_t run = 1; + +/** + * @brief Signal termination of program + */ +static void stop(int sig) { + run = 0; +} + + +/** + * @returns 1 if all bytes are printable, else 0. + */ +static int is_printable(const char *buf, size_t size) { + size_t i; + + for (i = 0; i < size; i++) + if (!isprint((int)buf[i])) + return 0; + + return 1; +} + +/** + * @brief Process a received record. + * + * Replace this with your own handling. The return value tells the caller how + * to acknowledge the record: + * 0 success -> ACCEPT (done, never redeliver it) + * > 0 transient failure -> RELEASE (retry later, maybe another consumer) + * < 0 permanent failure -> REJECT (give up; the broker archives it) + */ +static int process_message(const rd_kafka_message_t *rkm) { + printf("Message on %s [%" PRId32 "] at offset %" PRId64 + " (delivery count: %" PRId16 ")", + rd_kafka_topic_name(rkm->rkt), rkm->partition, rkm->offset, + rd_kafka_message_delivery_count(rkm)); + + if (rkm->key && is_printable(rkm->key, rkm->key_len)) + printf(" Key: %.*s", (int)rkm->key_len, (const char *)rkm->key); + + if (rkm->payload && is_printable(rkm->payload, rkm->len)) + printf(" Value: %.*s", (int)rkm->len, + (const char *)rkm->payload); + + printf("\n"); + + return 0; +} + + +int main(int argc, char **argv) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_kafka_resp_err_t err; + char errstr[512]; + const char *brokers; + const char *groupid; + char **topics; + int topic_cnt; + rd_kafka_topic_partition_list_t *subscription; + int i; + int ret = 0; /* Process exit code */ + + if (argc < 4) { + fprintf(stderr, + "%% Usage: " + "%s [topic2 ..]\n", + argv[0]); + return 1; + } + + brokers = argv[1]; + groupid = argv[2]; + topics = &argv[3]; + topic_cnt = argc - 3; + + conf = rd_kafka_conf_new(); + + if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + if (rd_kafka_conf_set(conf, "group.id", groupid, errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + if (rd_kafka_conf_set(conf, "share.acknowledgement.mode", "explicit", + errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { + fprintf(stderr, "%s\n", errstr); + rd_kafka_conf_destroy(conf); + return 1; + } + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + if (!rkshare) { + fprintf(stderr, "%% Failed to create new share consumer: %s\n", + errstr); + return 1; + } + + conf = NULL; + + subscription = rd_kafka_topic_partition_list_new(topic_cnt); + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(subscription, topics[i], + RD_KAFKA_PARTITION_UA); + + err = rd_kafka_share_subscribe(rkshare, subscription); + if (err) { + fprintf(stderr, "%% Failed to subscribe to %d topics: %s\n", + subscription->cnt, rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subscription); + rd_kafka_share_destroy(rkshare); + return 1; + } + + fprintf(stderr, + "%% Subscribed to %d topic(s), " + "waiting for rebalance and messages...\n", + subscription->cnt); + + rd_kafka_topic_partition_list_destroy(subscription); + + signal(SIGINT, stop); + + while (run) { + rd_kafka_messages_t *rkmessages = NULL; + rd_kafka_topic_partition_list_t *committed = NULL; + rd_kafka_error_t *error; + size_t rcvd_msgs; + size_t k; + int j; + + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + int fatal = rd_kafka_error_is_fatal(error); + fprintf(stderr, "%% Consume error%s: %s\n", + fatal ? " (fatal)" : "", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(rkmessages); + /* A fatal error is unrecoverable: stop consuming. */ + if (fatal) { + ret = 1; + goto done; + } + continue; + } + + rcvd_msgs = rd_kafka_messages_count(rkmessages); + if (rcvd_msgs == 0) { + rd_kafka_messages_destroy(rkmessages); + continue; + } + + printf("Received %zu messages\n", rcvd_msgs); + + for (k = 0; k < rcvd_msgs; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(rkmessages, k); + rd_kafka_share_AcknowledgeType_t ack_type; + int rc; + + if (rkm->err) { + /* A record delivered with an error has already + * been acknowledged for you by the library: + * RELEASE (put back for retry) for + * decompression errors, and REJECT (give up) + * for CRC or unsupported-format errors. You can + * still override that by acknowledging the + * offset yourself if your application needs + * different handling. */ + fprintf(stderr, "%% Consumer error: %d: %s\n", + rkm->err, rd_kafka_message_errstr(rkm)); + continue; + } + + /* Decide how to acknowledge based on the processing + * outcome: ACCEPT on success, RELEASE on a transient + * failure so the record can be retried, REJECT on a + * permanent failure so the broker archives it. */ + rc = process_message(rkm); + if (rc == 0) + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + else if (rc > 0) + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE; + else + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT; + + err = rd_kafka_share_acknowledge_type(rkshare, rkm, + ack_type); + if (err) + fprintf(stderr, + "%% Acknowledge error for " + "%s [%" PRId32 "] @ %" PRId64 ": %s\n", + rd_kafka_topic_name(rkm->rkt), + rkm->partition, rkm->offset, + rd_kafka_err2str(err)); + } + + /* Flush this batch's acknowledgements before the next poll(). + * commit_sync() blocks for the broker reply and reports the + * per-partition outcome. */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &committed); + if (error) { + fprintf(stderr, "%% Commit sync error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } else if (committed) { + for (j = 0; j < committed->cnt; j++) { + rd_kafka_topic_partition_t *rktpar = + &committed->elems[j]; + if (rktpar->err) + fprintf(stderr, + "%% Commit failed for " + "%s [%" PRId32 "]: %s\n", + rktpar->topic, + rktpar->partition, + rd_kafka_err2str(rktpar->err)); + } + rd_kafka_topic_partition_list_destroy(committed); + } + + rd_kafka_messages_destroy(rkmessages); + } + +done: + fprintf(stderr, "%% Closing share consumer\n"); + rd_kafka_share_consumer_close(rkshare); + + rd_kafka_share_destroy(rkshare); + + return ret; +} diff --git a/lib/librdkafka-2.10.1/examples/transactions-older-broker.c b/lib/librdkafka-2.15.0/examples/transactions-older-broker.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/transactions-older-broker.c rename to lib/librdkafka-2.15.0/examples/transactions-older-broker.c diff --git a/lib/librdkafka-2.10.1/examples/transactions.c b/lib/librdkafka-2.15.0/examples/transactions.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/transactions.c rename to lib/librdkafka-2.15.0/examples/transactions.c diff --git a/lib/librdkafka-2.10.1/examples/user_scram.c b/lib/librdkafka-2.15.0/examples/user_scram.c similarity index 100% rename from lib/librdkafka-2.10.1/examples/user_scram.c rename to lib/librdkafka-2.15.0/examples/user_scram.c diff --git a/lib/librdkafka-2.10.1/examples/win_ssl_cert_store.cpp b/lib/librdkafka-2.15.0/examples/win_ssl_cert_store.cpp similarity index 100% rename from lib/librdkafka-2.10.1/examples/win_ssl_cert_store.cpp rename to lib/librdkafka-2.15.0/examples/win_ssl_cert_store.cpp diff --git a/lib/librdkafka-2.10.1/lds-gen.py b/lib/librdkafka-2.15.0/lds-gen.py similarity index 100% rename from lib/librdkafka-2.10.1/lds-gen.py rename to lib/librdkafka-2.15.0/lds-gen.py diff --git a/lib/librdkafka-2.10.1/mainpage.doxy b/lib/librdkafka-2.15.0/mainpage.doxy similarity index 100% rename from lib/librdkafka-2.10.1/mainpage.doxy rename to lib/librdkafka-2.15.0/mainpage.doxy diff --git a/lib/librdkafka-2.10.1/mklove/.gitignore b/lib/librdkafka-2.15.0/mklove/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/mklove/.gitignore rename to lib/librdkafka-2.15.0/mklove/.gitignore diff --git a/lib/librdkafka-2.10.1/mklove/Makefile.base b/lib/librdkafka-2.15.0/mklove/Makefile.base similarity index 100% rename from lib/librdkafka-2.10.1/mklove/Makefile.base rename to lib/librdkafka-2.15.0/mklove/Makefile.base diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.atomics b/lib/librdkafka-2.15.0/mklove/modules/configure.atomics similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.atomics rename to lib/librdkafka-2.15.0/mklove/modules/configure.atomics diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.base b/lib/librdkafka-2.15.0/mklove/modules/configure.base similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.base rename to lib/librdkafka-2.15.0/mklove/modules/configure.base diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.builtin b/lib/librdkafka-2.15.0/mklove/modules/configure.builtin similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.builtin rename to lib/librdkafka-2.15.0/mklove/modules/configure.builtin diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.cc b/lib/librdkafka-2.15.0/mklove/modules/configure.cc similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.cc rename to lib/librdkafka-2.15.0/mklove/modules/configure.cc diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.cxx b/lib/librdkafka-2.15.0/mklove/modules/configure.cxx similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.cxx rename to lib/librdkafka-2.15.0/mklove/modules/configure.cxx diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.fileversion b/lib/librdkafka-2.15.0/mklove/modules/configure.fileversion similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.fileversion rename to lib/librdkafka-2.15.0/mklove/modules/configure.fileversion diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.gitversion b/lib/librdkafka-2.15.0/mklove/modules/configure.gitversion similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.gitversion rename to lib/librdkafka-2.15.0/mklove/modules/configure.gitversion diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.good_cflags b/lib/librdkafka-2.15.0/mklove/modules/configure.good_cflags similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.good_cflags rename to lib/librdkafka-2.15.0/mklove/modules/configure.good_cflags diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.host b/lib/librdkafka-2.15.0/mklove/modules/configure.host similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.host rename to lib/librdkafka-2.15.0/mklove/modules/configure.host diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.lib b/lib/librdkafka-2.15.0/mklove/modules/configure.lib similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.lib rename to lib/librdkafka-2.15.0/mklove/modules/configure.lib diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.libcurl b/lib/librdkafka-2.15.0/mklove/modules/configure.libcurl similarity index 95% rename from lib/librdkafka-2.10.1/mklove/modules/configure.libcurl rename to lib/librdkafka-2.15.0/mklove/modules/configure.libcurl index 174a4d8b52a..22508a2eaf7 100644 --- a/lib/librdkafka-2.10.1/mklove/modules/configure.libcurl +++ b/lib/librdkafka-2.15.0/mklove/modules/configure.libcurl @@ -45,8 +45,8 @@ void foo (void) { function install_source { local name=$1 local destdir=$2 - local ver=8.10.1 - local checksum="d15ebab765d793e2e96db090f0e172d127859d78ca6f6391d7eafecfd894bbc0" + local ver=8.20.0 + local checksum="fc5819cad3f9f5482669adcdc49a782c15f36d2a0715b395b06d9173593d2dc0" echo "### Installing $name $ver from source to $destdir" if [[ ! -f Makefile ]]; then @@ -87,7 +87,7 @@ function install_source { --disable-ldap{,s} \ --disable-libcurl-option \ --without-{librtmp,libidn2,winidn,nghttp2,nghttp3,ngtcp2,quiche,brotli,libpsl} && - time make CPPFLAGS="$CPPFLAGS" -j && + time make -j && make DESTDIR="${destdir}" prefix=/usr install local ret=$? diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.libsasl2 b/lib/librdkafka-2.15.0/mklove/modules/configure.libsasl2 similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.libsasl2 rename to lib/librdkafka-2.15.0/mklove/modules/configure.libsasl2 diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.libssl b/lib/librdkafka-2.15.0/mklove/modules/configure.libssl similarity index 95% rename from lib/librdkafka-2.10.1/mklove/modules/configure.libssl rename to lib/librdkafka-2.15.0/mklove/modules/configure.libssl index 3ef5cab7d08..0223a6949bf 100644 --- a/lib/librdkafka-2.10.1/mklove/modules/configure.libssl +++ b/lib/librdkafka-2.15.0/mklove/modules/configure.libssl @@ -91,9 +91,9 @@ function manual_checks { function libcrypto_install_source { local name=$1 local destdir=$2 - local ver=3.0.15 - local checksum="23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533" - local url=https://www.openssl.org/source/openssl-${ver}.tar.gz + local ver=3.5.6 + local checksum="deae7c80cba99c4b4f940ecadb3c3338b13cb77418409238e57d7f31f2a3b736" + local url=https://github.com/openssl/openssl/releases/download/openssl-${ver}/openssl-${ver}.tar.gz local conf_args="--prefix=/usr --openssldir=/usr/lib/ssl no-shared no-zlib" diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.libzstd b/lib/librdkafka-2.15.0/mklove/modules/configure.libzstd similarity index 93% rename from lib/librdkafka-2.10.1/mklove/modules/configure.libzstd rename to lib/librdkafka-2.15.0/mklove/modules/configure.libzstd index 9c26e07b27f..06aa1862985 100644 --- a/lib/librdkafka-2.10.1/mklove/modules/configure.libzstd +++ b/lib/librdkafka-2.15.0/mklove/modules/configure.libzstd @@ -42,8 +42,8 @@ void foo (void) { function install_source { local name=$1 local destdir=$2 - local ver=1.5.6 - local checksum="8c29e06cf42aacc1eafc4077ae2ec6c6fcb96a626157e0593d5e82a34fd403c1" + local ver=1.5.7 + local checksum="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" echo "### Installing $name $ver from source to $destdir" if [[ ! -f Makefile ]]; then diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.parseversion b/lib/librdkafka-2.15.0/mklove/modules/configure.parseversion similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.parseversion rename to lib/librdkafka-2.15.0/mklove/modules/configure.parseversion diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.pic b/lib/librdkafka-2.15.0/mklove/modules/configure.pic similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.pic rename to lib/librdkafka-2.15.0/mklove/modules/configure.pic diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.socket b/lib/librdkafka-2.15.0/mklove/modules/configure.socket similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/configure.socket rename to lib/librdkafka-2.15.0/mklove/modules/configure.socket diff --git a/lib/librdkafka-2.10.1/mklove/modules/configure.zlib b/lib/librdkafka-2.15.0/mklove/modules/configure.zlib similarity index 92% rename from lib/librdkafka-2.10.1/mklove/modules/configure.zlib rename to lib/librdkafka-2.15.0/mklove/modules/configure.zlib index f6df7bc62f7..dfd200757dc 100644 --- a/lib/librdkafka-2.10.1/mklove/modules/configure.zlib +++ b/lib/librdkafka-2.15.0/mklove/modules/configure.zlib @@ -42,8 +42,8 @@ void foo (void) { function install_source { local name=$1 local destdir=$2 - local ver=1.3.1 - local checksum="9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" + local ver=1.3.2 + local checksum="bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16" echo "### Installing $name $ver from source to $destdir" if [[ ! -f Makefile ]]; then diff --git a/lib/librdkafka-2.10.1/mklove/modules/patches/README.md b/lib/librdkafka-2.15.0/mklove/modules/patches/README.md similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/patches/README.md rename to lib/librdkafka-2.15.0/mklove/modules/patches/README.md diff --git a/lib/librdkafka-2.15.0/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch b/lib/librdkafka-2.15.0/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch new file mode 100644 index 00000000000..c9f47105991 --- /dev/null +++ b/lib/librdkafka-2.15.0/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch @@ -0,0 +1,11 @@ +--- a/configure 2026-05-14 23:31:11.123796321 +0200 ++++ b/configure 2026-05-14 23:32:59.800743981 +0200 +@@ -36043,7 +36043,7 @@ + + + +- if test "$cross_compiling" != "yes"; then ++ if false; then + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking runtime libs availability" >&5 + printf %s "checking runtime libs availability... " >&6; } diff --git a/lib/librdkafka-2.10.1/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch b/lib/librdkafka-2.15.0/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch similarity index 100% rename from lib/librdkafka-2.10.1/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch rename to lib/librdkafka-2.15.0/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch diff --git a/lib/librdkafka-2.10.1/packaging/RELEASE.md b/lib/librdkafka-2.15.0/packaging/RELEASE.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/RELEASE.md rename to lib/librdkafka-2.15.0/packaging/RELEASE.md diff --git a/lib/librdkafka-2.10.1/packaging/alpine/build-alpine.sh b/lib/librdkafka-2.15.0/packaging/alpine/build-alpine.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/alpine/build-alpine.sh rename to lib/librdkafka-2.15.0/packaging/alpine/build-alpine.sh diff --git a/lib/librdkafka-2.10.1/packaging/archlinux/PKGBUILD b/lib/librdkafka-2.15.0/packaging/archlinux/PKGBUILD similarity index 100% rename from lib/librdkafka-2.10.1/packaging/archlinux/PKGBUILD rename to lib/librdkafka-2.15.0/packaging/archlinux/PKGBUILD diff --git a/lib/librdkafka-2.10.1/packaging/cmake/Config.cmake.in b/lib/librdkafka-2.15.0/packaging/cmake/Config.cmake.in similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/Config.cmake.in rename to lib/librdkafka-2.15.0/packaging/cmake/Config.cmake.in diff --git a/lib/librdkafka-2.10.1/packaging/cmake/Modules/FindLZ4.cmake b/lib/librdkafka-2.15.0/packaging/cmake/Modules/FindLZ4.cmake similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/Modules/FindLZ4.cmake rename to lib/librdkafka-2.15.0/packaging/cmake/Modules/FindLZ4.cmake diff --git a/lib/librdkafka-2.10.1/packaging/cmake/Modules/FindZSTD.cmake b/lib/librdkafka-2.15.0/packaging/cmake/Modules/FindZSTD.cmake similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/Modules/FindZSTD.cmake rename to lib/librdkafka-2.15.0/packaging/cmake/Modules/FindZSTD.cmake diff --git a/lib/librdkafka-2.10.1/packaging/cmake/Modules/LICENSE.FindZstd b/lib/librdkafka-2.15.0/packaging/cmake/Modules/LICENSE.FindZstd similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/Modules/LICENSE.FindZstd rename to lib/librdkafka-2.15.0/packaging/cmake/Modules/LICENSE.FindZstd diff --git a/lib/librdkafka-2.10.1/packaging/cmake/README.md b/lib/librdkafka-2.15.0/packaging/cmake/README.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/README.md rename to lib/librdkafka-2.15.0/packaging/cmake/README.md diff --git a/lib/librdkafka-2.10.1/packaging/cmake/config.h.in b/lib/librdkafka-2.15.0/packaging/cmake/config.h.in similarity index 93% rename from lib/librdkafka-2.10.1/packaging/cmake/config.h.in rename to lib/librdkafka-2.15.0/packaging/cmake/config.h.in index 9e356c5f9ac..d09a56f94a5 100644 --- a/lib/librdkafka-2.10.1/packaging/cmake/config.h.in +++ b/lib/librdkafka-2.15.0/packaging/cmake/config.h.in @@ -3,6 +3,7 @@ #cmakedefine01 ENABLE_REFCNT_DEBUG #cmakedefine01 HAVE_ATOMICS_32 +#cmakedefine01 HAVE_ATOMICS_32_ATOMIC #cmakedefine01 HAVE_ATOMICS_32_SYNC #if (HAVE_ATOMICS_32) @@ -14,6 +15,7 @@ #endif #cmakedefine01 HAVE_ATOMICS_64 +#cmakedefine01 HAVE_ATOMICS_64_ATOMIC #cmakedefine01 HAVE_ATOMICS_64_SYNC #if (HAVE_ATOMICS_64) @@ -43,6 +45,7 @@ #cmakedefine01 HAVE_REGEX #cmakedefine01 HAVE_STRNDUP #cmakedefine01 HAVE_RAND_R +#cmakedefine01 HAVE_GETENTROPY #cmakedefine01 HAVE_PTHREAD_SETNAME_GNU #cmakedefine01 HAVE_PTHREAD_SETNAME_DARWIN #cmakedefine01 HAVE_PTHREAD_SETNAME_FREEBSD diff --git a/lib/librdkafka-2.10.1/packaging/cmake/parseversion.cmake b/lib/librdkafka-2.15.0/packaging/cmake/parseversion.cmake similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/parseversion.cmake rename to lib/librdkafka-2.15.0/packaging/cmake/parseversion.cmake diff --git a/lib/librdkafka-2.10.1/packaging/cmake/rdkafka.pc.in b/lib/librdkafka-2.15.0/packaging/cmake/rdkafka.pc.in similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/rdkafka.pc.in rename to lib/librdkafka-2.15.0/packaging/cmake/rdkafka.pc.in diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/atomic_32_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/atomic_32_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/atomic_32_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/atomic_32_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/atomic_64_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/atomic_64_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/atomic_64_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/atomic_64_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/c11threads_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/c11threads_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/c11threads_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/c11threads_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/crc32c_hw_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/crc32c_hw_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/crc32c_hw_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/crc32c_hw_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/dlopen_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/dlopen_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/dlopen_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/dlopen_test.c diff --git a/lib/librdkafka-2.15.0/packaging/cmake/try_compile/getentropy_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/getentropy_test.c new file mode 100644 index 00000000000..f32594349fe --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/getentropy_test.c @@ -0,0 +1,7 @@ +#include +#include + +int main() { + char seed[16]; + return getentropy(seed, sizeof(seed)); +} diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/libsasl2_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/libsasl2_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/libsasl2_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/libsasl2_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/pthread_setname_darwin_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/pthread_setname_darwin_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/pthread_setname_darwin_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/pthread_setname_darwin_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/pthread_setname_freebsd_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/pthread_setname_freebsd_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/pthread_setname_freebsd_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/pthread_setname_freebsd_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/pthread_setname_gnu_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/pthread_setname_gnu_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/pthread_setname_gnu_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/pthread_setname_gnu_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/rand_r_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/rand_r_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/rand_r_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/rand_r_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/rdkafka_setup.cmake b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/rdkafka_setup.cmake similarity index 85% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/rdkafka_setup.cmake rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/rdkafka_setup.cmake index 5ea7f7dc6c4..ca9bc93e904 100644 --- a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/rdkafka_setup.cmake +++ b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/rdkafka_setup.cmake @@ -16,6 +16,12 @@ try_compile( "${TRYCOMPILE_SRC_DIR}/rand_r_test.c" ) +try_compile( + HAVE_GETENTROPY + "${CMAKE_CURRENT_BINARY_DIR}/try_compile" + "${TRYCOMPILE_SRC_DIR}/getentropy_test.c" +) + try_compile( HAVE_PTHREAD_SETNAME_GNU "${CMAKE_CURRENT_BINARY_DIR}/try_compile" @@ -42,6 +48,7 @@ try_compile( # Atomic 32 tests { set(LINK_ATOMIC NO) set(HAVE_ATOMICS_32 NO) +set(HAVE_ATOMICS_32_ATOMIC NO) set(HAVE_ATOMICS_32_SYNC NO) try_compile( @@ -51,7 +58,7 @@ try_compile( ) if(_atomics_32) - set(HAVE_ATOMICS_32 YES) + set(HAVE_ATOMICS_32_ATOMIC YES) else() try_compile( _atomics_32_lib @@ -60,7 +67,7 @@ else() LINK_LIBRARIES "-latomic" ) if(_atomics_32_lib) - set(HAVE_ATOMICS_32 YES) + set(HAVE_ATOMICS_32_ATOMIC YES) set(LINK_ATOMIC YES) else() try_compile( @@ -70,10 +77,14 @@ else() ) endif() endif() +if (HAVE_ATOMICS_32_ATOMIC OR HAVE_ATOMICS_32_SYNC) + set(HAVE_ATOMICS_32 YES) +endif() # } # Atomic 64 tests { set(HAVE_ATOMICS_64 NO) +set(HAVE_ATOMICS_64_ATOMIC NO) set(HAVE_ATOMICS_64_SYNC NO) try_compile( @@ -83,7 +94,7 @@ try_compile( ) if(_atomics_64) - set(HAVE_ATOMICS_64 YES) + set(HAVE_ATOMICS_64_ATOMIC YES) else() try_compile( _atomics_64_lib @@ -92,7 +103,7 @@ else() LINK_LIBRARIES "-latomic" ) if(_atomics_64_lib) - set(HAVE_ATOMICS_64 YES) + set(HAVE_ATOMICS_64_ATOMIC YES) set(LINK_ATOMIC YES) else() try_compile( @@ -102,6 +113,9 @@ else() ) endif() endif() +if (HAVE_ATOMICS_64_ATOMIC OR HAVE_ATOMICS_64_SYNC) + set(HAVE_ATOMICS_64 YES) +endif() # } # C11 threads diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/regex_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/regex_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/regex_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/regex_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/strndup_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/strndup_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/strndup_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/strndup_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/sync_32_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/sync_32_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/sync_32_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/sync_32_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cmake/try_compile/sync_64_test.c b/lib/librdkafka-2.15.0/packaging/cmake/try_compile/sync_64_test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cmake/try_compile/sync_64_test.c rename to lib/librdkafka-2.15.0/packaging/cmake/try_compile/sync_64_test.c diff --git a/lib/librdkafka-2.10.1/packaging/cp/README.md b/lib/librdkafka-2.15.0/packaging/cp/README.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cp/README.md rename to lib/librdkafka-2.15.0/packaging/cp/README.md diff --git a/lib/librdkafka-2.10.1/packaging/cp/check_features.c b/lib/librdkafka-2.15.0/packaging/cp/check_features.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cp/check_features.c rename to lib/librdkafka-2.15.0/packaging/cp/check_features.c diff --git a/lib/librdkafka-2.10.1/packaging/cp/verify-deb.sh b/lib/librdkafka-2.15.0/packaging/cp/verify-deb.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cp/verify-deb.sh rename to lib/librdkafka-2.15.0/packaging/cp/verify-deb.sh diff --git a/lib/librdkafka-2.10.1/packaging/cp/verify-packages.sh b/lib/librdkafka-2.15.0/packaging/cp/verify-packages.sh similarity index 93% rename from lib/librdkafka-2.10.1/packaging/cp/verify-packages.sh rename to lib/librdkafka-2.15.0/packaging/cp/verify-packages.sh index 02230dc8b00..41bcb18c424 100755 --- a/lib/librdkafka-2.10.1/packaging/cp/verify-packages.sh +++ b/lib/librdkafka-2.15.0/packaging/cp/verify-packages.sh @@ -54,11 +54,10 @@ verify_debian_distros() { local platform=$1 local version=$2 echo "#### Verifying Debian packages for $platform ####" - # Last Debian 10 version is 2.5.0 - verify_debian debian:10 "2.5.0" verify_debian debian:11 $version verify_debian debian:12 $version - verify_debian ubuntu:20.04 $version + # Last Ubuntu 20 version is 2.13.2 + verify_debian ubuntu:20.04 "2.13.2" verify_debian ubuntu:22.04 $version verify_debian ubuntu:24.04 $version } diff --git a/lib/librdkafka-2.10.1/packaging/cp/verify-rpm.sh b/lib/librdkafka-2.15.0/packaging/cp/verify-rpm.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/cp/verify-rpm.sh rename to lib/librdkafka-2.15.0/packaging/cp/verify-rpm.sh diff --git a/lib/librdkafka-2.10.1/packaging/debian/.gitignore b/lib/librdkafka-2.15.0/packaging/debian/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/.gitignore rename to lib/librdkafka-2.15.0/packaging/debian/.gitignore diff --git a/lib/librdkafka-2.10.1/packaging/debian/changelog b/lib/librdkafka-2.15.0/packaging/debian/changelog similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/changelog rename to lib/librdkafka-2.15.0/packaging/debian/changelog diff --git a/lib/librdkafka-2.10.1/packaging/debian/compat b/lib/librdkafka-2.15.0/packaging/debian/compat similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/compat rename to lib/librdkafka-2.15.0/packaging/debian/compat diff --git a/lib/librdkafka-2.10.1/packaging/debian/control b/lib/librdkafka-2.15.0/packaging/debian/control similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/control rename to lib/librdkafka-2.15.0/packaging/debian/control diff --git a/lib/librdkafka-2.10.1/packaging/debian/copyright b/lib/librdkafka-2.15.0/packaging/debian/copyright similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/copyright rename to lib/librdkafka-2.15.0/packaging/debian/copyright diff --git a/lib/librdkafka-2.10.1/packaging/debian/docs b/lib/librdkafka-2.15.0/packaging/debian/docs similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/docs rename to lib/librdkafka-2.15.0/packaging/debian/docs diff --git a/lib/librdkafka-2.10.1/packaging/debian/gbp.conf b/lib/librdkafka-2.15.0/packaging/debian/gbp.conf similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/gbp.conf rename to lib/librdkafka-2.15.0/packaging/debian/gbp.conf diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.dirs b/lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.dirs similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.dirs rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.dirs diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.examples b/lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.examples similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.examples rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.examples diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.install b/lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.install similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.install rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.install diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.substvars b/lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.substvars similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka-dev.substvars rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka-dev.substvars diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka.dsc b/lib/librdkafka-2.15.0/packaging/debian/librdkafka.dsc similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka.dsc rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka.dsc diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka1-dbg.substvars b/lib/librdkafka-2.15.0/packaging/debian/librdkafka1-dbg.substvars similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka1-dbg.substvars rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka1-dbg.substvars diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka1.dirs b/lib/librdkafka-2.15.0/packaging/debian/librdkafka1.dirs similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka1.dirs rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka1.dirs diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka1.install b/lib/librdkafka-2.15.0/packaging/debian/librdkafka1.install similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka1.install rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka1.install diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka1.postinst.debhelper b/lib/librdkafka-2.15.0/packaging/debian/librdkafka1.postinst.debhelper similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka1.postinst.debhelper rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka1.postinst.debhelper diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka1.postrm.debhelper b/lib/librdkafka-2.15.0/packaging/debian/librdkafka1.postrm.debhelper similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka1.postrm.debhelper rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka1.postrm.debhelper diff --git a/lib/librdkafka-2.10.1/packaging/debian/librdkafka1.symbols b/lib/librdkafka-2.15.0/packaging/debian/librdkafka1.symbols similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/librdkafka1.symbols rename to lib/librdkafka-2.15.0/packaging/debian/librdkafka1.symbols diff --git a/lib/librdkafka-2.10.1/packaging/debian/rules b/lib/librdkafka-2.15.0/packaging/debian/rules similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/rules rename to lib/librdkafka-2.15.0/packaging/debian/rules diff --git a/lib/librdkafka-2.10.1/packaging/debian/source/format b/lib/librdkafka-2.15.0/packaging/debian/source/format similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/source/format rename to lib/librdkafka-2.15.0/packaging/debian/source/format diff --git a/lib/librdkafka-2.10.1/packaging/debian/watch b/lib/librdkafka-2.15.0/packaging/debian/watch similarity index 100% rename from lib/librdkafka-2.10.1/packaging/debian/watch rename to lib/librdkafka-2.15.0/packaging/debian/watch diff --git a/lib/librdkafka-2.10.1/packaging/get_version.py b/lib/librdkafka-2.15.0/packaging/get_version.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/get_version.py rename to lib/librdkafka-2.15.0/packaging/get_version.py diff --git a/lib/librdkafka-2.10.1/packaging/homebrew/README.md b/lib/librdkafka-2.15.0/packaging/homebrew/README.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/homebrew/README.md rename to lib/librdkafka-2.15.0/packaging/homebrew/README.md diff --git a/lib/librdkafka-2.10.1/packaging/homebrew/brew-update-pr.sh b/lib/librdkafka-2.15.0/packaging/homebrew/brew-update-pr.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/homebrew/brew-update-pr.sh rename to lib/librdkafka-2.15.0/packaging/homebrew/brew-update-pr.sh diff --git a/lib/librdkafka-2.15.0/packaging/macos/build-release-artifacts.sh b/lib/librdkafka-2.15.0/packaging/macos/build-release-artifacts.sh new file mode 100755 index 00000000000..7bc59b3ee60 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/macos/build-release-artifacts.sh @@ -0,0 +1,26 @@ +./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip +make -j all examples check +examples/rdkafka_example -X builtin.features +otool -L src/librdkafka.dylib +otool -L src-cpp/librdkafka++.dylib +make -j -C tests build +export TEST_CONSUMER_GROUP_PROTOCOL=classic +make -C tests run_local_quick +export TEST_CONSUMER_GROUP_PROTOCOL=consumer +# Skip tests needing special limits +TESTS_WITH_INCREASED_NLIMIT="0153" +export TESTS_SKIP="$TESTS_WITH_INCREASED_NLIMIT" +make -C tests run_local_quick +# Now run only those tests with different limits + +# Tests needing increased number of file descriptors +PREV_N=$(ulimit -n) +ulimit -n 2048 +export TESTS_SKIP="" +export TESTS="$TESTS_WITH_INCREASED_NLIMIT" +make -C tests run_local_quick +ulimit -n $PREV_N + + +DESTDIR="$PWD/dest" make install +(cd dest && tar cvzf ../artifacts/librdkafka.tgz .) diff --git a/lib/librdkafka-2.10.1/packaging/mingw-w64/configure-build-msys2-mingw-static.sh b/lib/librdkafka-2.15.0/packaging/mingw-w64/configure-build-msys2-mingw-static.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/mingw-w64/configure-build-msys2-mingw-static.sh rename to lib/librdkafka-2.15.0/packaging/mingw-w64/configure-build-msys2-mingw-static.sh diff --git a/lib/librdkafka-2.10.1/packaging/mingw-w64/configure-build-msys2-mingw.sh b/lib/librdkafka-2.15.0/packaging/mingw-w64/configure-build-msys2-mingw.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/mingw-w64/configure-build-msys2-mingw.sh rename to lib/librdkafka-2.15.0/packaging/mingw-w64/configure-build-msys2-mingw.sh diff --git a/lib/librdkafka-2.10.1/packaging/mingw-w64/travis-before-install.sh b/lib/librdkafka-2.15.0/packaging/mingw-w64/export-variables.sh similarity index 53% rename from lib/librdkafka-2.10.1/packaging/mingw-w64/travis-before-install.sh rename to lib/librdkafka-2.15.0/packaging/mingw-w64/export-variables.sh index e75507f9339..822a9d3d5fb 100644 --- a/lib/librdkafka-2.10.1/packaging/mingw-w64/travis-before-install.sh +++ b/lib/librdkafka-2.15.0/packaging/mingw-w64/export-variables.sh @@ -8,13 +8,6 @@ export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start' export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --" export msys2+=" -msys2 -c "\"\$@"\" --" -# Have to update pacman first or choco upgrade will failure due to migration -# to zstd instead of xz compression -$msys2 pacman -Sy --noconfirm pacman - -## Install more MSYS2 packages from https://packages.msys2.org/base here -$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc mingw-w64-x86_64-make mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-lz4 mingw-w64-x86_64-zstd - taskkill //IM gpg-agent.exe //F || true # https://travis-ci.community/t/4967 export PATH=/C/msys64/mingw64/bin:$PATH export MAKE=mingw32-make # so that Autotools can find it diff --git a/lib/librdkafka-2.15.0/packaging/mingw-w64/run-tests.sh b/lib/librdkafka-2.15.0/packaging/mingw-w64/run-tests.sh new file mode 100644 index 00000000000..413e613e069 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/mingw-w64/run-tests.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +cd tests +export TEST_CONSUMER_GROUP_PROTOCOL=classic +./test-runner.exe -l -Q -p1 +export TEST_CONSUMER_GROUP_PROTOCOL=consumer +./test-runner.exe -l -Q -p1 diff --git a/lib/librdkafka-2.10.1/packaging/mingw-w64/semaphoreci-build.sh b/lib/librdkafka-2.15.0/packaging/mingw-w64/semaphoreci-build.sh similarity index 90% rename from lib/librdkafka-2.10.1/packaging/mingw-w64/semaphoreci-build.sh rename to lib/librdkafka-2.15.0/packaging/mingw-w64/semaphoreci-build.sh index 378545b443a..461cd8a7aa7 100644 --- a/lib/librdkafka-2.10.1/packaging/mingw-w64/semaphoreci-build.sh +++ b/lib/librdkafka-2.15.0/packaging/mingw-w64/semaphoreci-build.sh @@ -17,7 +17,7 @@ fi archive="${PWD}/$1" -source ./packaging/mingw-w64/travis-before-install.sh +source ./packaging/mingw-w64/export-variables.sh if [[ $linkage == "static" ]]; then ./packaging/mingw-w64/configure-build-msys2-mingw-static.sh diff --git a/lib/librdkafka-2.10.1/packaging/nuget/.gitignore b/lib/librdkafka-2.15.0/packaging/nuget/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/.gitignore rename to lib/librdkafka-2.15.0/packaging/nuget/.gitignore diff --git a/lib/librdkafka-2.10.1/packaging/nuget/README.md b/lib/librdkafka-2.15.0/packaging/nuget/README.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/README.md rename to lib/librdkafka-2.15.0/packaging/nuget/README.md diff --git a/lib/librdkafka-2.10.1/packaging/nuget/artifact.py b/lib/librdkafka-2.15.0/packaging/nuget/artifact.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/artifact.py rename to lib/librdkafka-2.15.0/packaging/nuget/artifact.py diff --git a/lib/librdkafka-2.10.1/packaging/nuget/cleanup-s3.py b/lib/librdkafka-2.15.0/packaging/nuget/cleanup-s3.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/cleanup-s3.py rename to lib/librdkafka-2.15.0/packaging/nuget/cleanup-s3.py diff --git a/lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip b/lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip rename to lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr120.zip diff --git a/lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip b/lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip rename to lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-win32__bldtype-Release/msvcr140.zip diff --git a/lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip b/lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip rename to lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr120.zip diff --git a/lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip b/lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip rename to lib/librdkafka-2.15.0/packaging/nuget/common/p-common__plat-windows__arch-x64__bldtype-Release/msvcr140.zip diff --git a/lib/librdkafka-2.10.1/packaging/nuget/nuget.sh b/lib/librdkafka-2.15.0/packaging/nuget/nuget.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/nuget.sh rename to lib/librdkafka-2.15.0/packaging/nuget/nuget.sh diff --git a/lib/librdkafka-2.10.1/packaging/nuget/nugetpackage.py b/lib/librdkafka-2.15.0/packaging/nuget/nugetpackage.py similarity index 92% rename from lib/librdkafka-2.10.1/packaging/nuget/nugetpackage.py rename to lib/librdkafka-2.15.0/packaging/nuget/nugetpackage.py index ab3655782fd..e31f2d92fd8 100644 --- a/lib/librdkafka-2.10.1/packaging/nuget/nugetpackage.py +++ b/lib/librdkafka-2.15.0/packaging/nuget/nugetpackage.py @@ -91,6 +91,14 @@ class NugetPackage (Package): 'librdkafka.tgz', './usr/local/lib/librdkafka.so.1', 'runtimes/linux-arm64/native/librdkafka.so'), + # Linux glibc centos8 s390x without GSSAPI (no external deps) + Mapping({'arch': 's390x', + 'plat': 'linux', + 'dist': 'centos8', + 'lnk': 'all'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka.so.1', + 'runtimes/linux-s390x/native/librdkafka.so'), # Linux musl alpine x64 without GSSAPI (no external deps) Mapping({'arch': 'x64', @@ -100,6 +108,14 @@ class NugetPackage (Package): 'librdkafka.tgz', './usr/local/lib/librdkafka.so.1', 'runtimes/linux-x64/native/alpine-librdkafka.so'), + # Linux musl alpine arm64 without GSSAPI (no external deps) + Mapping({'arch': 'arm64', + 'plat': 'linux', + 'dist': 'alpine', + 'lnk': 'all'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka.so.1', + 'runtimes/linux-arm64/native/alpine-librdkafka.so'), # Common Win runtime Mapping({'arch': 'x64', @@ -136,8 +152,8 @@ class NugetPackage (Package): Mapping({'arch': 'x64', 'plat': 'win'}, 'librdkafka.redist*', - 'build/native/bin/v142/x64/Release/zlib1.dll', - 'runtimes/win-x64/native/zlib1.dll'), + 'build/native/bin/v142/x64/Release/z.dll', + 'runtimes/win-x64/native/z.dll'), Mapping({'arch': 'x64', 'plat': 'win'}, 'librdkafka.redist*', @@ -197,8 +213,8 @@ class NugetPackage (Package): Mapping({'arch': 'x86', 'plat': 'win'}, 'librdkafka.redist*', - 'build/native/bin/v142/Win32/Release/zlib1.dll', - 'runtimes/win-x86/native/zlib1.dll'), + 'build/native/bin/v142/Win32/Release/z.dll', + 'runtimes/win-x86/native/z.dll'), Mapping({'arch': 'x86', 'plat': 'win'}, 'librdkafka.redist*', diff --git a/lib/librdkafka-2.10.1/packaging/nuget/packaging.py b/lib/librdkafka-2.15.0/packaging/nuget/packaging.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/packaging.py rename to lib/librdkafka-2.15.0/packaging/nuget/packaging.py diff --git a/lib/librdkafka-2.10.1/packaging/nuget/push-to-nuget.sh b/lib/librdkafka-2.15.0/packaging/nuget/push-to-nuget.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/push-to-nuget.sh rename to lib/librdkafka-2.15.0/packaging/nuget/push-to-nuget.sh diff --git a/lib/librdkafka-2.10.1/packaging/nuget/release.py b/lib/librdkafka-2.15.0/packaging/nuget/release.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/release.py rename to lib/librdkafka-2.15.0/packaging/nuget/release.py diff --git a/lib/librdkafka-2.10.1/packaging/nuget/requirements.txt b/lib/librdkafka-2.15.0/packaging/nuget/requirements.txt similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/requirements.txt rename to lib/librdkafka-2.15.0/packaging/nuget/requirements.txt diff --git a/lib/librdkafka-2.10.1/packaging/nuget/staticpackage.py b/lib/librdkafka-2.15.0/packaging/nuget/staticpackage.py similarity index 90% rename from lib/librdkafka-2.10.1/packaging/nuget/staticpackage.py rename to lib/librdkafka-2.15.0/packaging/nuget/staticpackage.py index 9a555eb32ec..68fe3964a88 100644 --- a/lib/librdkafka-2.10.1/packaging/nuget/staticpackage.py +++ b/lib/librdkafka-2.15.0/packaging/nuget/staticpackage.py @@ -72,6 +72,24 @@ class StaticPackage (Package): './usr/local/lib/pkgconfig/rdkafka-static.pc', 'librdkafka_glibc_linux_arm64.pc'), + # glibc linux s390x static lib and pkg-config file + Mapping({'arch': 's390x', + 'plat': 'linux', + 'dist': 'centos8', + 'lnk': 'all', + '!extra': 'gssapi'}, + 'librdkafka.tgz', + './usr/local/lib/librdkafka-static.a', + 'librdkafka_glibc_linux_s390x.a'), + Mapping({'arch': 's390x', + 'plat': 'linux', + 'dist': 'centos8', + 'lnk': 'all', + '!extra': 'gssapi'}, + 'librdkafka.tgz', + './usr/local/lib/pkgconfig/rdkafka-static.pc', + 'librdkafka_glibc_linux_s390x.pc'), + # musl linux static lib and pkg-config file Mapping({'arch': 'x64', 'plat': 'linux', diff --git a/lib/librdkafka-2.10.1/packaging/nuget/templates/librdkafka.redist.nuspec b/lib/librdkafka-2.15.0/packaging/nuget/templates/librdkafka.redist.nuspec similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/templates/librdkafka.redist.nuspec rename to lib/librdkafka-2.15.0/packaging/nuget/templates/librdkafka.redist.nuspec diff --git a/lib/librdkafka-2.10.1/packaging/nuget/templates/librdkafka.redist.props b/lib/librdkafka-2.15.0/packaging/nuget/templates/librdkafka.redist.props similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/templates/librdkafka.redist.props rename to lib/librdkafka-2.15.0/packaging/nuget/templates/librdkafka.redist.props diff --git a/lib/librdkafka-2.10.1/packaging/nuget/templates/librdkafka.redist.targets b/lib/librdkafka-2.15.0/packaging/nuget/templates/librdkafka.redist.targets similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/templates/librdkafka.redist.targets rename to lib/librdkafka-2.15.0/packaging/nuget/templates/librdkafka.redist.targets diff --git a/lib/librdkafka-2.10.1/packaging/nuget/zfile/__init__.py b/lib/librdkafka-2.15.0/packaging/nuget/zfile/__init__.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/zfile/__init__.py rename to lib/librdkafka-2.15.0/packaging/nuget/zfile/__init__.py diff --git a/lib/librdkafka-2.10.1/packaging/nuget/zfile/zfile.py b/lib/librdkafka-2.15.0/packaging/nuget/zfile/zfile.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/nuget/zfile/zfile.py rename to lib/librdkafka-2.15.0/packaging/nuget/zfile/zfile.py diff --git a/lib/librdkafka-2.10.1/packaging/rpm/.gitignore b/lib/librdkafka-2.15.0/packaging/rpm/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/.gitignore rename to lib/librdkafka-2.15.0/packaging/rpm/.gitignore diff --git a/lib/librdkafka-2.10.1/packaging/rpm/Makefile b/lib/librdkafka-2.15.0/packaging/rpm/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/Makefile rename to lib/librdkafka-2.15.0/packaging/rpm/Makefile diff --git a/lib/librdkafka-2.10.1/packaging/rpm/README.md b/lib/librdkafka-2.15.0/packaging/rpm/README.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/README.md rename to lib/librdkafka-2.15.0/packaging/rpm/README.md diff --git a/lib/librdkafka-2.10.1/packaging/rpm/el7-x86_64.cfg b/lib/librdkafka-2.15.0/packaging/rpm/el7-x86_64.cfg similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/el7-x86_64.cfg rename to lib/librdkafka-2.15.0/packaging/rpm/el7-x86_64.cfg diff --git a/lib/librdkafka-2.10.1/packaging/rpm/librdkafka.spec b/lib/librdkafka-2.15.0/packaging/rpm/librdkafka.spec similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/librdkafka.spec rename to lib/librdkafka-2.15.0/packaging/rpm/librdkafka.spec diff --git a/lib/librdkafka-2.10.1/packaging/rpm/mock-on-docker.sh b/lib/librdkafka-2.15.0/packaging/rpm/mock-on-docker.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/mock-on-docker.sh rename to lib/librdkafka-2.15.0/packaging/rpm/mock-on-docker.sh diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/.gitignore b/lib/librdkafka-2.15.0/packaging/rpm/tests/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/.gitignore rename to lib/librdkafka-2.15.0/packaging/rpm/tests/.gitignore diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/Makefile b/lib/librdkafka-2.15.0/packaging/rpm/tests/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/Makefile rename to lib/librdkafka-2.15.0/packaging/rpm/tests/Makefile diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/README.md b/lib/librdkafka-2.15.0/packaging/rpm/tests/README.md similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/README.md rename to lib/librdkafka-2.15.0/packaging/rpm/tests/README.md diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/run-test.sh b/lib/librdkafka-2.15.0/packaging/rpm/tests/run-test.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/run-test.sh rename to lib/librdkafka-2.15.0/packaging/rpm/tests/run-test.sh diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/test-on-docker.sh b/lib/librdkafka-2.15.0/packaging/rpm/tests/test-on-docker.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/test-on-docker.sh rename to lib/librdkafka-2.15.0/packaging/rpm/tests/test-on-docker.sh diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/test.c b/lib/librdkafka-2.15.0/packaging/rpm/tests/test.c similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/test.c rename to lib/librdkafka-2.15.0/packaging/rpm/tests/test.c diff --git a/lib/librdkafka-2.10.1/packaging/rpm/tests/test.cpp b/lib/librdkafka-2.15.0/packaging/rpm/tests/test.cpp similarity index 100% rename from lib/librdkafka-2.10.1/packaging/rpm/tests/test.cpp rename to lib/librdkafka-2.15.0/packaging/rpm/tests/test.cpp diff --git a/lib/librdkafka-2.15.0/packaging/tools/Dockerfile b/lib/librdkafka-2.15.0/packaging/tools/Dockerfile new file mode 100644 index 00000000000..d29ba1db579 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/Dockerfile @@ -0,0 +1,32 @@ +# hadolint global ignore=DL3003,DL3008,DL3009,DL3013,DL3042 +FROM ubuntu:24.04 + +ARG UID + +RUN NEEDRESTART_MODE=a apt-get update && apt-get install -y \ + wget curl python3 git openjdk-21-jdk-headless python3-venv \ + python3-setuptools build-essential libssl-dev libcurl4-openssl-dev \ + zlib1g-dev libsasl2-dev libzstd-dev clang-format-18 doxygen graphviz + +RUN wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb && \ + dpkg -i rapidjson-dev.deb && \ + rm rapidjson-dev.deb + +RUN mkdir -p /home/user +WORKDIR /home/user +COPY ./tests/trivup tests/trivup +COPY ./tests/requirements.txt tests/requirements.txt +COPY ./packaging/tools/requirements.txt packaging/tools/requirements.txt +COPY ./packaging/nuget/requirements.txt packaging/nuget/requirements.txt +RUN chown -R ${UID} /home/user + +USER ${UID} + +USER ${UID} +RUN python3 -m venv venv && \ + . venv/bin/activate && \ + pip install -U pip && \ + (cd tests && pip install -r requirements.txt) && \ + pip install -r packaging/tools/requirements.txt && \ + pip install -r packaging/nuget/requirements.txt && \ + rm -rf tests packaging diff --git a/lib/librdkafka-2.15.0/packaging/tools/Dockerfile.alpine b/lib/librdkafka-2.15.0/packaging/tools/Dockerfile.alpine new file mode 100644 index 00000000000..1cf359327ef --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/Dockerfile.alpine @@ -0,0 +1,9 @@ +FROM alpine:3.16.9 + +ARG UID + +RUN apk add bash gcc g++ make cmake git bsd-compat-headers +RUN mkdir -p /home/user +WORKDIR /home/user +RUN chown -R ${UID} /home/user +USER ${UID} diff --git a/lib/librdkafka-2.15.0/packaging/tools/Dockerfile.manylinux b/lib/librdkafka-2.15.0/packaging/tools/Dockerfile.manylinux new file mode 100644 index 00000000000..319b5c933c1 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/Dockerfile.manylinux @@ -0,0 +1,8 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 + +ARG UID + +RUN mkdir -p /home/user +WORKDIR /home/user +RUN chown -R ${UID} /home/user +USER ${UID} diff --git a/lib/librdkafka-2.15.0/packaging/tools/build-configurations-checks.sh b/lib/librdkafka-2.15.0/packaging/tools/build-configurations-checks.sh new file mode 100755 index 00000000000..76b32cf121c --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/build-configurations-checks.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -e +build_tool="$1" +docker_image_tag="$2" + +if [ -n "$docker_image_tag" ]; then + echo "Running in docker image tag: $docker_image_tag" + # Running on the host, spin up the docker builder. + ./packaging/tools/run-in-docker.sh $docker_image_tag ./packaging/tools/build-configurations-checks.sh $build_tool + # Only reached on exec error + exit $? +fi + +if [ -z "$build_tool" ]; then + # Default to using make if no build tool is specified. + build_tool="make" +fi + +# Clone the repo so other builds are unaffected of what we're doing +# and we get a pristine build tree. +git clone /librdkafka /home/user/librdkafka + +cd /home/user/librdkafka + +# Disable all flags to make sure it +# compiles correctly in all cases +if [ "$build_tool" = "make" ]; then +./configure --disable-ssl --disable-gssapi \ +--disable-curl --disable-zlib \ +--disable-zstd --disable-lz4-ext --disable-regex-ext \ +--disable-c11threads --disable-syslog \ +--enable-werror --enable-devel +cat ./config.h +else +cmake -DWITH_SSL=OFF -DWITH_SASL_CYRUS=OFF \ + -DWITH_CURL=OFF -DWITH_ZLIB=OFF \ + -DWITH_ZSTD=OFF -DHAVE_REGEX=OFF -DWITH_C11THREADS=OFF \ + -DWITH_LIBDL=OFF +cat ./generated/config.h +fi +make -j + +export CI=true +if [ "0$build_tool" = "0make" ]; then +make -j -C tests run_local_quick +else +ctest -VV -R RdKafkaTestBrokerLessQuick --output-on-failure +fi diff --git a/lib/librdkafka-2.10.1/packaging/tools/build-deb-package.sh b/lib/librdkafka-2.15.0/packaging/tools/build-deb-package.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/build-deb-package.sh rename to lib/librdkafka-2.15.0/packaging/tools/build-deb-package.sh diff --git a/lib/librdkafka-2.10.1/packaging/tools/build-debian.sh b/lib/librdkafka-2.15.0/packaging/tools/build-debian.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/build-debian.sh rename to lib/librdkafka-2.15.0/packaging/tools/build-debian.sh diff --git a/lib/librdkafka-2.10.1/packaging/tools/build-manylinux.sh b/lib/librdkafka-2.15.0/packaging/tools/build-manylinux.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/build-manylinux.sh rename to lib/librdkafka-2.15.0/packaging/tools/build-manylinux.sh diff --git a/lib/librdkafka-2.15.0/packaging/tools/build-release-artifacts-s390x.sh b/lib/librdkafka-2.15.0/packaging/tools/build-release-artifacts-s390x.sh new file mode 100755 index 00000000000..578d8fd4c31 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/build-release-artifacts-s390x.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +set -e + +ON_S390X="$1" + +if [[ "$ON_S390X" != "--on-s390x" ]]; then + if [ -z "$S390X_USER" ]; then + echo "S390X_USER not defined"; + exit 1; + fi + if [ -z "$LOCAL_KEY" ]; then + echo "LOCAL_KEY not defined"; + exit 1; + fi + + SSH_KEY_PATH="v1/devel/kv/cp-env/s390x-key/IBM-Cloud-S390x-key" + SSH_PRIVATE_KEY_FIELD="private_key" + SSH_IP_FIELD="ip" + SSH_KNOWN_HOST="known_host" + S390X_HOST=$(vault kv get -field=$SSH_IP_FIELD $SSH_KEY_PATH) + SSH_USER_AT_HOST="$S390X_USER@$S390X_HOST" + SSH_COMMAND="ssh -o ServerAliveInterval=60 -i ./$LOCAL_KEY $SSH_USER_AT_HOST" + SCP_COMMAND="scp -i ./$LOCAL_KEY" + + vault kv get -field=$SSH_PRIVATE_KEY_FIELD $SSH_KEY_PATH > ./$LOCAL_KEY + chmod go-rwx ./$LOCAL_KEY + echo "SSH Key saved to $LOCAL_KEY" + + if [ -z "$(ssh-keygen -F $S390X_HOST)" ]; then + S390X_KNOWN_HOST=$(vault kv get -field=$SSH_KNOWN_HOST $SSH_KEY_PATH) + echo "$S390X_KNOWN_HOST" >> ~/.ssh/known_hosts; + echo "Added $S390X_HOST to the list of known hosts" + fi + + DIR=$(mktemp -d --suffix=librdkafka) + eval $SSH_COMMAND mkdir $DIR + eval $SCP_COMMAND ./packaging/tools/build-release-artifacts-s390x.sh $SSH_USER_AT_HOST:$DIR/build-release-artifacts-s390x.sh + echo "Running build on s390x" + CURRENT_TARGET=$(git symbolic-ref --short -q HEAD || git describe --tags --exact-match 2>/dev/null) + eval $SSH_COMMAND $DIR/build-release-artifacts-s390x.sh --on-s390x $CURRENT_TARGET "$@" + RET=$? + + if [ "x$RET" = "x0" ]; then + DEST_FILE=${@: -1} + mkdir -p $(dirname $DEST_FILE) + eval $SCP_COMMAND $SSH_USER_AT_HOST:$DIR/librdkafka/artifacts/librdkafka.tgz $DEST_FILE + RET=$? + echo "Copied artifact to Semaphore" + fi + if [[ "$DIR" =~ ^/tmp/.*$ ]]; then + eval $SSH_COMMAND rm -rf $DIR + RET=$? + if [ "$RET" != "0" ]; then + echo "Failed to remove build directory"; + else + echo "Removed build directory"; + fi + fi + exit $RET +fi + +export DEBIAN_FRONTEND=noninteractive +CURRENT_TARGET=$2 +DIR=$(dirname $0) +shift 2 +# Clean up stopped builds +find /tmp -maxdepth 1 -name "tmp.*librdkafka" -mtime +1 -exec rm -rf {} + +echo "ON S390x: Installing pre-requisites" +sudo apt update +sudo apt install -y git ca-certificates curl gnupg + +if ! command -v docker >/dev/null 2>&1; then + echo "Installing docker..." + # Add Docker's official GPG key: + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + sudo tee /etc/apt/sources.list.d/docker.sources < +# packaging/tools/build-release-artifacts.sh [--disable-gssapi] [--platform ] # # The output path must be a relative path and inside the librdkafka directory # structure. @@ -42,6 +43,12 @@ else disable_gssapi="" fi +docker_platform="" +if [ "$1" = "--platform" ]; then + docker_platform="--platform=$2" + shift 2 +fi + # Check if we're running on the host or the (docker) build target. if [ "$1" = "--in-docker" -a $# -eq 2 ]; then output="$2" @@ -49,13 +56,14 @@ elif [ $# -eq 2 ]; then docker_image="$1" output="$2" else - echo "Usage: $0 [--disable-gssapi] " + echo "Usage: $0 [--disable-gssapi] [--platform ] " exit 1 fi if [ -n "$docker_image" ]; then # Running on the host, spin up the docker builder. - exec docker run -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi --in-docker "/v/$output" + exec docker run --rm $docker_platform \ + -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi --in-docker "/v/$output" # Only reached on exec error exit $? fi @@ -76,7 +84,8 @@ if grep -q alpine /etc/os-release 2>/dev/null ; then else # CentOS - yum install -y libstdc++-devel gcc gcc-c++ python3 git perl-IPC-Cmd perl-Pod-Html $extra_pkgs_rpm + yum install -y libstdc++-devel gcc gcc-c++ python3 git perl-IPC-Cmd \ + perl-Pod-Html perl-bignum perl-Time-Piece $extra_pkgs_rpm fi @@ -123,6 +132,7 @@ fi # Run quick test suite, mark it as CI to avoid time/resource sensitive # tests to fail in case the worker is under-powered. CI=true make -C tests run_local_quick +CI=true TEST_CONSUMER_GROUP_PROTOCOL=consumer make -C tests run_local_quick # Install librdkafka and then make a tar ball of the installed files. diff --git a/lib/librdkafka-2.10.1/packaging/tools/distro-build.sh b/lib/librdkafka-2.15.0/packaging/tools/distro-build.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/distro-build.sh rename to lib/librdkafka-2.15.0/packaging/tools/distro-build.sh diff --git a/lib/librdkafka-2.10.1/packaging/tools/gh-release-checksums.py b/lib/librdkafka-2.15.0/packaging/tools/gh-release-checksums.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/gh-release-checksums.py rename to lib/librdkafka-2.15.0/packaging/tools/gh-release-checksums.py diff --git a/lib/librdkafka-2.10.1/packaging/tools/rdutcoverage.sh b/lib/librdkafka-2.15.0/packaging/tools/rdutcoverage.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/rdutcoverage.sh rename to lib/librdkafka-2.15.0/packaging/tools/rdutcoverage.sh diff --git a/lib/librdkafka-2.10.1/packaging/tools/requirements.txt b/lib/librdkafka-2.15.0/packaging/tools/requirements.txt similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/requirements.txt rename to lib/librdkafka-2.15.0/packaging/tools/requirements.txt diff --git a/lib/librdkafka-2.15.0/packaging/tools/run-in-docker.sh b/lib/librdkafka-2.15.0/packaging/tools/run-in-docker.sh new file mode 100755 index 00000000000..6fc21d55172 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/run-in-docker.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [...]" + exit 1 +fi + +IMAGE=$1 +SCRIPT_DIR=$(dirname "$0") +ENTRYPOINT=${2} +REST=${@:3} + +DOCKERFILE=Dockerfile +if [[ $IMAGE =~ ^"test-runner-manylinux-amd64-" ]]; then + DOCKERFILE=Dockerfile.manylinux +fi +if [[ $IMAGE =~ ^"test-runner-alpine-amd64-" ]]; then + DOCKERFILE=Dockerfile.alpine +fi + +if [ $(which cache) ]; then + cache restore ${IMAGE}.tar +fi +if [ ! -f ./${IMAGE}.tar ]; then + docker build -f $SCRIPT_DIR/$DOCKERFILE -t $IMAGE --build-arg UID=$UID . + docker save $IMAGE -o ./${IMAGE}.tar + + if [ $(which cache) ]; then + cache store ${IMAGE}.tar ./${IMAGE}.tar + fi +else + docker load -i ./${IMAGE}.tar +fi + +docker run --rm --entrypoint $ENTRYPOINT \ + -v .:/librdkafka -w /librdkafka -e CI -u $UID:$UID ${IMAGE} ${REST} diff --git a/lib/librdkafka-2.15.0/packaging/tools/run-integration-tests.sh b/lib/librdkafka-2.15.0/packaging/tools/run-integration-tests.sh new file mode 100755 index 00000000000..a63f36ea54b --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/run-integration-tests.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 []" + echo "Example: $0 3.9.0 7.9.0" + exit 1 +fi + +KAFKA_VERSION=$1 +CP_VERSION=$2 +if [ -n "$3" ]; then + export TEST_CONSUMER_GROUP_PROTOCOL=$3 +fi + +source /home/user/venv/bin/activate +./configure --install-deps --enable-werror --enable-devel +./packaging/tools/rdutcoverage.sh +make copyright-check +make -j all examples check +echo "Verifying that CONFIGURATION.md does not have manual changes" +git diff --exit-code CONFIGURATION.md +examples/rdkafka_example -X builtin.features +ldd src/librdkafka.so.1 +ldd src-cpp/librdkafka++.so.1 +make -j -C tests build +make -C tests run_local_quick +DESTDIR="$PWD/dest" make install +(cd tests && python3 -m trivup.clusters.KafkaCluster --kraft \ + --conf '["group.share.min.record.lock.duration.ms=1000"]' \ + --version ${KAFKA_VERSION} \ + --cpversion ${CP_VERSION} --cmd 'make quick') diff --git a/lib/librdkafka-2.15.0/packaging/tools/run-share-consumer-tests.sh b/lib/librdkafka-2.15.0/packaging/tools/run-share-consumer-tests.sh new file mode 100755 index 00000000000..f5de4e14f97 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/run-share-consumer-tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +# Use env vars matching run-all-tests.sh convention, +# with fallback to positional args for backward compatibility. +export TEST_KAFKA_GIT_REF="${TEST_KAFKA_GIT_REF:-${1:-4.3.0}}" +export TEST_CP_VERSION="${TEST_CP_VERSION:-${2:-8.2.0}}" +TEST_CONFIGURATION="${TEST_CONFIGURATION:---kraft}" + +if [[ "$(uname)" == "Darwin" ]]; then + CONFIGURE_ARGS="--install-deps --source-deps-only" +else + source /home/user/venv/bin/activate + CONFIGURE_ARGS="--install-deps --enable-werror" +fi +./configure ${CONFIGURE_ARGS} +make -j all +make -j -C tests build + +echo "arguments: $TEST_ARGS" +(cd tests && python3 -m trivup.clusters.KafkaCluster $TEST_CONFIGURATION \ + --conf '["group.share.min.record.lock.duration.ms=1000", "transaction.state.log.replication.factor=1", "transaction.state.log.min.isr=1"]' \ + --version "$TEST_KAFKA_GIT_REF" \ + --cpversion "$TEST_CP_VERSION" \ + --cmd "TESTS_SKIP_BEFORE=0155 python run-test-batches.py $TEST_ARGS") diff --git a/lib/librdkafka-2.15.0/packaging/tools/run-style-check.sh b/lib/librdkafka-2.15.0/packaging/tools/run-style-check.sh new file mode 100755 index 00000000000..f739d320cbb --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/run-style-check.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +source /home/user/venv/bin/activate +make style-check diff --git a/lib/librdkafka-2.15.0/packaging/tools/setup-share-telemetry-testing.sh b/lib/librdkafka-2.15.0/packaging/tools/setup-share-telemetry-testing.sh new file mode 100755 index 00000000000..738fe2badd0 --- /dev/null +++ b/lib/librdkafka-2.15.0/packaging/tools/setup-share-telemetry-testing.sh @@ -0,0 +1,140 @@ +#!/bin/bash +set -euo pipefail + +ROOT=$(git rev-parse --show-toplevel) +VERIFY_DIR=${ROOT}/tests/share_telemetry_verify +WORK_DIR=${VERIFY_DIR}/.work + +KAFKA_VERSION=${KAFKA_VERSION:-4.2.0} +KAFKA_DIR=${WORK_DIR}/kafka_2.13-${KAFKA_VERSION} +KAFKA_DATA=${WORK_DIR}/kafka-data +TUTORIALS_REPO=${WORK_DIR}/tutorials +PLUGIN_JAR=${WORK_DIR}/client-telemetry-reporter-plugin.jar + +export TELEMETRY_TOPIC=${TELEMETRY_TOPIC:-client-telemetry-metrics} +SUBSCRIPTION_NAME=share-consumer-ci + +mkdir -p ${WORK_DIR} + +if [ ! -f "${PLUGIN_JAR}" ]; then + echo "==> Building OTLP reporter plugin" + if [ ! -d "${TUTORIALS_REPO}" ]; then + git clone --depth 1 https://github.com/confluentinc/tutorials.git ${TUTORIALS_REPO} + fi + sed -i.bak \ + "s|id 'com.github.johnrengelman.shadow' version '8.1.1'|id 'com.gradleup.shadow' version '8.3.6'|" \ + ${TUTORIALS_REPO}/client-telemetry-reporter-plugin/kafka/build.gradle + (cd ${TUTORIALS_REPO} && ./gradlew --no-daemon clean \ + :client-telemetry-reporter-plugin:kafka:build) + cp ${TUTORIALS_REPO}/client-telemetry-reporter-plugin/kafka/build/libs/client-telemetry-reporter-plugin.jar \ + ${PLUGIN_JAR} +fi + +if [ ! -d "${KAFKA_DIR}" ]; then + echo "==> Setting up Kafka broker ${KAFKA_VERSION}" + wget -q https://downloads.apache.org/kafka/${KAFKA_VERSION}/kafka_2.13-${KAFKA_VERSION}.tgz \ + -O ${WORK_DIR}/kafka.tgz + tar -C ${WORK_DIR} -xzf ${WORK_DIR}/kafka.tgz + + SP=${KAFKA_DIR}/config/server.properties + sed -i.bak \ + -e 's|^listeners=PLAINTEXT://:9092,CONTROLLER://:9093|listeners=PLAINTEXT://:9092,DOCKER://:19092,CONTROLLER://:9093|' \ + -e 's|^advertised.listeners=PLAINTEXT://localhost:9092,CONTROLLER://localhost:9093|advertised.listeners=PLAINTEXT://localhost:9092,DOCKER://host.docker.internal:19092,CONTROLLER://localhost:9093|' \ + -e 's|^listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL|listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,DOCKER:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL|' \ + -e "s|^log.dirs=/tmp/kraft-combined-logs|log.dirs=${KAFKA_DATA}|" \ + ${SP} + echo 'metric.reporters=io.confluent.developer.ClientOtlpMetricsReporter' >> ${SP} + + cp ${PLUGIN_JAR} ${KAFKA_DIR}/libs/ +fi + +echo "==> Formatting fresh broker storage" +rm -rf ${KAFKA_DATA} +mkdir -p ${KAFKA_DATA} +CLUSTER_ID=$(${KAFKA_DIR}/bin/kafka-storage.sh random-uuid) +${KAFKA_DIR}/bin/kafka-storage.sh format --standalone \ + -t ${CLUSTER_ID} -c ${KAFKA_DIR}/config/server.properties + +TEST_CONF_BAK=${ROOT}/tests/test.conf.bak.$$ +LOG_DIR=${ROOT}/artifacts +cleanup() { + local rc=$? + set +e + mkdir -p ${LOG_DIR} + docker logs otel-collector > ${LOG_DIR}/otel-collector.log 2>&1 || true + if [ -f ${KAFKA_DIR}/logs/server.log ]; then + cp ${KAFKA_DIR}/logs/server.log ${LOG_DIR}/kafka-server.log + fi + ${KAFKA_DIR}/bin/kafka-server-stop.sh 2>/dev/null + docker stop otel-collector 2>/dev/null + docker rm otel-collector 2>/dev/null + if [ -f "${TEST_CONF_BAK}" ]; then + mv ${TEST_CONF_BAK} ${ROOT}/tests/test.conf + fi + exit ${rc} +} +trap cleanup EXIT + +echo "==> Starting OpenTelemetry Collector" +docker rm -f otel-collector 2>/dev/null || true +docker run -d --name otel-collector \ + --add-host=host.docker.internal:host-gateway \ + -e TELEMETRY_TOPIC=${TELEMETRY_TOPIC} \ + -p 4317:4317 \ + -v ${VERIFY_DIR}/share-otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml \ + otel/opentelemetry-collector-contrib + +echo "==> Starting Kafka broker" +OTEL_EXPORTER_OTLP_ENDPOINT=127.0.0.1:4317 \ + ${KAFKA_DIR}/bin/kafka-server-start.sh -daemon \ + ${KAFKA_DIR}/config/server.properties + +BROKER_UP=0 +for i in $(seq 1 60); do + if ${KAFKA_DIR}/bin/kafka-broker-api-versions.sh \ + --bootstrap-server localhost:9092 > /dev/null 2>&1; then + BROKER_UP=1 + break + fi + sleep 1 +done + +if [ ${BROKER_UP} -eq 0 ]; then + echo "ERROR: Kafka broker did not come up within 60s on localhost:9092" + echo " see kafka-server.log (uploaded as artifact) for details" + exit 1 +fi + +echo "==> Creating telemetry sink topic and metrics subscription" +${KAFKA_DIR}/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create \ + --topic ${TELEMETRY_TOPIC} --partitions 1 --replication-factor 1 --if-not-exists +${KAFKA_DIR}/bin/kafka-client-metrics.sh --bootstrap-server localhost:9092 --alter \ + --name ${SUBSCRIPTION_NAME} \ + --metrics "org.apache.kafka.consumer.share." \ + --interval 5000 + +echo "==> Building librdkafka" +(cd ${ROOT} && ./configure --install-deps --enable-werror --enable-devel) +make -j -C ${ROOT} +make -j -C ${ROOT}/tests build + +if [ -f ${ROOT}/tests/test.conf ]; then + cp ${ROOT}/tests/test.conf ${TEST_CONF_BAK} +fi +cat > ${ROOT}/tests/test.conf <<'EOF' +metadata.broker.list=localhost:9092 +security.protocol=plaintext +enable.metrics.push=true +EOF + +echo "==> Pre-test infra sanity check" +if ! ${KAFKA_DIR}/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ + --describe --topic ${TELEMETRY_TOPIC} > /dev/null 2>&1; then + echo "ERROR: telemetry topic '${TELEMETRY_TOPIC}' missing or broker unreachable" + echo " broker may have died during the build phase" + echo " see kafka-server.log / otel-collector.log artifacts" + exit 1 +fi + +echo "==> Running share consumer telemetry test (0190)" +(cd ${ROOT}/tests && TESTS=0190 ./run-test.sh) diff --git a/lib/librdkafka-2.10.1/packaging/tools/style-format.sh b/lib/librdkafka-2.15.0/packaging/tools/style-format.sh similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/style-format.sh rename to lib/librdkafka-2.15.0/packaging/tools/style-format.sh diff --git a/lib/librdkafka-2.10.1/packaging/tools/update_rpcs_max_versions.py b/lib/librdkafka-2.15.0/packaging/tools/update_rpcs_max_versions.py similarity index 100% rename from lib/librdkafka-2.10.1/packaging/tools/update_rpcs_max_versions.py rename to lib/librdkafka-2.15.0/packaging/tools/update_rpcs_max_versions.py diff --git a/lib/librdkafka-2.10.1/service.yml b/lib/librdkafka-2.15.0/service.yml similarity index 92% rename from lib/librdkafka-2.10.1/service.yml rename to lib/librdkafka-2.15.0/service.yml index 9c8da7969be..aa038b1d27b 100644 --- a/lib/librdkafka-2.10.1/service.yml +++ b/lib/librdkafka-2.15.0/service.yml @@ -8,14 +8,24 @@ github: semaphore: enable: true pipeline_enable: false + pipeline_files: + # Additional pipeline files run to consider for status checks + - path: .semaphore/semaphore-integration.yml + level: pipeline triggers: - tags - branches + - pull_requests branches: - master + # Semaphore pipeline changes go here - /semaphore.*/ + # Prefix branches this way to run the CI before opening a PR - /dev_.*/ + # Feature branches for new features still in development - /feature\/.*/ + # Release branches for backporting fixes + - /release\/v\d+\.\d+\.x$/ tasks: - name: verify-linux-packages branch: master diff --git a/lib/librdkafka-2.10.1/src-cpp/CMakeLists.txt b/lib/librdkafka-2.15.0/src-cpp/CMakeLists.txt similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/CMakeLists.txt rename to lib/librdkafka-2.15.0/src-cpp/CMakeLists.txt diff --git a/lib/librdkafka-2.10.1/src-cpp/ConfImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/ConfImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/ConfImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/ConfImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/ConsumerImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/ConsumerImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/ConsumerImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/ConsumerImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/HandleImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/HandleImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/HandleImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/HandleImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/HeadersImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/HeadersImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/HeadersImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/HeadersImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/KafkaConsumerImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/KafkaConsumerImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/KafkaConsumerImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/KafkaConsumerImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/Makefile b/lib/librdkafka-2.15.0/src-cpp/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/Makefile rename to lib/librdkafka-2.15.0/src-cpp/Makefile diff --git a/lib/librdkafka-2.10.1/src-cpp/MessageImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/MessageImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/MessageImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/MessageImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/MetadataImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/MetadataImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/MetadataImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/MetadataImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/ProducerImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/ProducerImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/ProducerImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/ProducerImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/QueueImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/QueueImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/QueueImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/QueueImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/README.md b/lib/librdkafka-2.15.0/src-cpp/README.md similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/README.md rename to lib/librdkafka-2.15.0/src-cpp/README.md diff --git a/lib/librdkafka-2.10.1/src-cpp/RdKafka.cpp b/lib/librdkafka-2.15.0/src-cpp/RdKafka.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/RdKafka.cpp rename to lib/librdkafka-2.15.0/src-cpp/RdKafka.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/TopicImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/TopicImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/TopicImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/TopicImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/TopicPartitionImpl.cpp b/lib/librdkafka-2.15.0/src-cpp/TopicPartitionImpl.cpp similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/TopicPartitionImpl.cpp rename to lib/librdkafka-2.15.0/src-cpp/TopicPartitionImpl.cpp diff --git a/lib/librdkafka-2.10.1/src-cpp/rdkafkacpp.h b/lib/librdkafka-2.15.0/src-cpp/rdkafkacpp.h similarity index 99% rename from lib/librdkafka-2.10.1/src-cpp/rdkafkacpp.h rename to lib/librdkafka-2.15.0/src-cpp/rdkafkacpp.h index 3da10a39ecf..71ec3ddf4c1 100644 --- a/lib/librdkafka-2.10.1/src-cpp/rdkafkacpp.h +++ b/lib/librdkafka-2.15.0/src-cpp/rdkafkacpp.h @@ -112,7 +112,7 @@ namespace RdKafka { * @remark This value should only be used during compile time, * for runtime checks of version use RdKafka::version() */ -#define RD_KAFKA_VERSION 0x020a01ff +#define RD_KAFKA_VERSION 0x020f00ff /** * @brief Returns the librdkafka version as integer. @@ -327,6 +327,11 @@ enum ErrorCode { ERR__AUTO_OFFSET_RESET = -140, /** Partition log truncation detected */ ERR__LOG_TRUNCATION = -139, + /** A different record in the batch was invalid + * and this message failed persisting. */ + ERR__INVALID_DIFFERENT_RECORD = -138, + /** Broker is going away but client isn't terminating */ + ERR__DESTROY_BROKER = -137, /** End internal error codes */ ERR__END = -100, @@ -544,7 +549,28 @@ enum ErrorCode { /** Unable to update finalized features due to server error */ ERR_FEATURE_UPDATE_FAILED = 96, /** Request principal deserialization failed during forwarding */ - ERR_PRINCIPAL_DESERIALIZATION_FAILURE = 97 + ERR_PRINCIPAL_DESERIALIZATION_FAILURE = 97, + /** Unknown Topic Id */ + ERR_UNKNOWN_TOPIC_ID = 100, + /** The member epoch is fenced by the group coordinator */ + ERR_FENCED_MEMBER_EPOCH = 110, + /** The instance ID is still used by another member in the + * consumer group */ + ERR_UNRELEASED_INSTANCE_ID = 111, + /** The assignor or its version range is not supported by the consumer + * group */ + ERR_UNSUPPORTED_ASSIGNOR = 112, + /** The member epoch is stale */ + ERR_STALE_MEMBER_EPOCH = 113, + /** Client sent a push telemetry request with an invalid or outdated + * subscription ID. */ + ERR_UNKNOWN_SUBSCRIPTION_ID = 117, + /** Client sent a push telemetry request larger than the maximum size + * the broker will accept. */ + ERR_TELEMETRY_TOO_LARGE = 118, + /** Client metadata is stale, + * client should rebootstrap to obtain new metadata. */ + ERR_REBOOTSTRAP_REQUIRED = 129 }; diff --git a/lib/librdkafka-2.10.1/src-cpp/rdkafkacpp_int.h b/lib/librdkafka-2.15.0/src-cpp/rdkafkacpp_int.h similarity index 100% rename from lib/librdkafka-2.10.1/src-cpp/rdkafkacpp_int.h rename to lib/librdkafka-2.15.0/src-cpp/rdkafkacpp_int.h diff --git a/lib/librdkafka-2.10.1/src/CMakeLists.txt b/lib/librdkafka-2.15.0/src/CMakeLists.txt similarity index 98% rename from lib/librdkafka-2.10.1/src/CMakeLists.txt rename to lib/librdkafka-2.15.0/src/CMakeLists.txt index bbe63cff481..2754ca765d5 100644 --- a/lib/librdkafka-2.10.1/src/CMakeLists.txt +++ b/lib/librdkafka-2.15.0/src/CMakeLists.txt @@ -26,6 +26,7 @@ set( rdkafka_offset.c rdkafka_op.c rdkafka_partition.c + rdkafka_share_acknowledgement.c rdkafka_pattern.c rdkafka_queue.c rdkafka_range_assignor.c @@ -51,6 +52,7 @@ set( rdkafka_mock.c rdkafka_mock_handlers.c rdkafka_mock_cgrp.c + rdkafka_mock_sharegrp.c rdkafka_error.c rdkafka_fetcher.c rdkafka_telemetry.c @@ -70,6 +72,9 @@ set( rdregex.c rdstring.c rdunittest.c + rdunittest_acknowledge.c + rdunittest_fetcher.c + rdunittest_msgset_errors.c rdvarint.c rdmap.c snappy.c diff --git a/lib/librdkafka-2.10.1/src/Makefile b/lib/librdkafka-2.15.0/src/Makefile similarity index 91% rename from lib/librdkafka-2.10.1/src/Makefile rename to lib/librdkafka-2.15.0/src/Makefile index 0d0635ce30e..4a2223fe147 100644 --- a/lib/librdkafka-2.10.1/src/Makefile +++ b/lib/librdkafka-2.15.0/src/Makefile @@ -41,7 +41,8 @@ SRCS= rdkafka.c rdkafka_broker.c rdkafka_msg.c rdkafka_topic.c \ rdkafka_conf.c rdkafka_timer.c rdkafka_offset.c \ rdkafka_transport.c rdkafka_buf.c rdkafka_queue.c rdkafka_op.c \ rdkafka_request.c rdkafka_cgrp.c rdkafka_pattern.c \ - rdkafka_partition.c rdkafka_subscription.c \ + rdkafka_partition.c rdkafka_share_acknowledgement.c \ + rdkafka_subscription.c \ rdkafka_assignment.c \ rdkafka_assignor.c rdkafka_range_assignor.c \ rdkafka_roundrobin_assignor.c rdkafka_sticky_assignor.c \ @@ -56,8 +57,9 @@ SRCS= rdkafka.c rdkafka_broker.c rdkafka_msg.c rdkafka_topic.c \ rdkafka_header.c rdkafka_admin.c rdkafka_aux.c \ rdkafka_background.c rdkafka_idempotence.c rdkafka_cert.c \ rdkafka_txnmgr.c rdkafka_coord.c rdbase64.c \ - rdvarint.c rdbuf.c rdmap.c rdunittest.c \ - rdkafka_mock.c rdkafka_mock_handlers.c rdkafka_mock_cgrp.c \ + rdvarint.c rdbuf.c rdmap.c rdunittest.c rdunittest_fetcher.c \ + rdunittest_acknowledge.c rdunittest_msgset_errors.c \ + rdkafka_mock.c rdkafka_mock_handlers.c rdkafka_mock_cgrp.c rdkafka_mock_sharegrp.c \ rdkafka_error.c rdkafka_fetcher.c rdkafka_telemetry.c \ rdkafka_telemetry_encode.c rdkafka_telemetry_decode.c \ nanopb/pb_encode.c nanopb/pb_decode.c nanopb/pb_common.c \ diff --git a/lib/librdkafka-2.10.1/src/README.lz4.md b/lib/librdkafka-2.15.0/src/README.lz4.md similarity index 100% rename from lib/librdkafka-2.10.1/src/README.lz4.md rename to lib/librdkafka-2.15.0/src/README.lz4.md diff --git a/lib/librdkafka-2.15.0/src/cJSON.c b/lib/librdkafka-2.15.0/src/cJSON.c new file mode 100644 index 00000000000..23b6ee64301 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/cJSON.c @@ -0,0 +1,3191 @@ +/* + Copyright (c) 2009-2017 Dave Gamble and cJSON contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +/* cJSON */ +/* JSON parser in C. */ + +/* disable warnings about old C89 functions in MSVC */ +#if !defined(_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) +#define _CRT_SECURE_NO_DEPRECATE +#endif + +#ifdef __GNUC__ +#pragma GCC visibility push(default) +#endif +#if defined(_MSC_VER) +#pragma warning (push) +/* disable warning about single line comments in system headers */ +#pragma warning (disable : 4001) +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef ENABLE_LOCALES +#include +#endif + +#if defined(_MSC_VER) +#pragma warning (pop) +#endif +#ifdef __GNUC__ +#pragma GCC visibility pop +#endif + +#include "cJSON.h" + +/* define our own boolean type */ +#ifdef true +#undef true +#endif +#define true ((cJSON_bool)1) + +#ifdef false +#undef false +#endif +#define false ((cJSON_bool)0) + +/* define isnan and isinf for ANSI C, if in C99 or above, isnan and isinf has been defined in math.h */ +#ifndef isinf +#define isinf(d) (isnan((d - d)) && !isnan(d)) +#endif +#ifndef isnan +#define isnan(d) (d != d) +#endif + +#ifndef NAN +#ifdef _WIN32 +#define NAN sqrt(-1.0) +#else +#define NAN 0.0/0.0 +#endif +#endif + +typedef struct { + const unsigned char *json; + size_t position; +} error; +static error global_error = { NULL, 0 }; + +CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) +{ + return (const char*) (global_error.json + global_error.position); +} + +CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) +{ + if (!cJSON_IsString(item)) + { + return NULL; + } + + return item->valuestring; +} + +CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) +{ + if (!cJSON_IsNumber(item)) + { + return (double) NAN; + } + + return item->valuedouble; +} + +/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 19) + #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. +#endif + +CJSON_PUBLIC(const char*) cJSON_Version(void) +{ + static char version[15]; + sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); + + return version; +} + +/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */ +static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2) +{ + if ((string1 == NULL) || (string2 == NULL)) + { + return 1; + } + + if (string1 == string2) + { + return 0; + } + + for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++) + { + if (*string1 == '\0') + { + return 0; + } + } + + return tolower(*string1) - tolower(*string2); +} + +typedef struct internal_hooks +{ + void *(CJSON_CDECL *allocate)(size_t size); + void (CJSON_CDECL *deallocate)(void *pointer); + void *(CJSON_CDECL *reallocate)(void *pointer, size_t size); +} internal_hooks; + +#if defined(_MSC_VER) +/* work around MSVC error C2322: '...' address of dllimport '...' is not static */ +static void * CJSON_CDECL internal_malloc(size_t size) +{ + return malloc(size); +} +static void CJSON_CDECL internal_free(void *pointer) +{ + free(pointer); +} +static void * CJSON_CDECL internal_realloc(void *pointer, size_t size) +{ + return realloc(pointer, size); +} +#else +#define internal_malloc malloc +#define internal_free free +#define internal_realloc realloc +#endif + +/* strlen of character literals resolved at compile time */ +#define static_strlen(string_literal) (sizeof(string_literal) - sizeof("")) + +static internal_hooks global_hooks = { internal_malloc, internal_free, internal_realloc }; + +static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks) +{ + size_t length = 0; + unsigned char *copy = NULL; + + if (string == NULL) + { + return NULL; + } + + length = strlen((const char*)string) + sizeof(""); + copy = (unsigned char*)hooks->allocate(length); + if (copy == NULL) + { + return NULL; + } + memcpy(copy, string, length); + + return copy; +} + +CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) +{ + if (hooks == NULL) + { + /* Reset hooks */ + global_hooks.allocate = malloc; + global_hooks.deallocate = free; + global_hooks.reallocate = realloc; + return; + } + + global_hooks.allocate = malloc; + if (hooks->malloc_fn != NULL) + { + global_hooks.allocate = hooks->malloc_fn; + } + + global_hooks.deallocate = free; + if (hooks->free_fn != NULL) + { + global_hooks.deallocate = hooks->free_fn; + } + + /* use realloc only if both free and malloc are used */ + global_hooks.reallocate = NULL; + if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) + { + global_hooks.reallocate = realloc; + } +} + +/* Internal constructor. */ +static cJSON *cJSON_New_Item(const internal_hooks * const hooks) +{ + cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON)); + if (node) + { + memset(node, '\0', sizeof(cJSON)); + } + + return node; +} + +/* Delete a cJSON structure. */ +CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) +{ + cJSON *next = NULL; + while (item != NULL) + { + next = item->next; + if (!(item->type & cJSON_IsReference) && (item->child != NULL)) + { + cJSON_Delete(item->child); + } + if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) + { + global_hooks.deallocate(item->valuestring); + item->valuestring = NULL; + } + if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) + { + global_hooks.deallocate(item->string); + item->string = NULL; + } + global_hooks.deallocate(item); + item = next; + } +} + +/* get the decimal point character of the current locale */ +static unsigned char get_decimal_point(void) +{ +#ifdef ENABLE_LOCALES + struct lconv *lconv = localeconv(); + return (unsigned char) lconv->decimal_point[0]; +#else + return '.'; +#endif +} + +typedef struct +{ + const unsigned char *content; + size_t length; + size_t offset; + size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */ + internal_hooks hooks; +} parse_buffer; + +/* check if the given size is left to read in a given parse buffer (starting with 1) */ +#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) +/* check if the buffer can be accessed at the given index (starting with 0) */ +#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) +#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) +/* get a pointer to the buffer at the position */ +#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) + +/* Parse the input text to generate a number, and populate the result into item. */ +static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer) +{ + double number = 0; + unsigned char *after_end = NULL; + unsigned char *number_c_string; + unsigned char decimal_point = get_decimal_point(); + size_t i = 0; + size_t number_string_length = 0; + cJSON_bool has_decimal_point = false; + + if ((input_buffer == NULL) || (input_buffer->content == NULL)) + { + return false; + } + + /* copy the number into a temporary buffer and replace '.' with the decimal point + * of the current locale (for strtod) + * This also takes care of '\0' not necessarily being available for marking the end of the input */ + for (i = 0; can_access_at_index(input_buffer, i); i++) + { + switch (buffer_at_offset(input_buffer)[i]) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '+': + case '-': + case 'e': + case 'E': + number_string_length++; + break; + + case '.': + number_string_length++; + has_decimal_point = true; + break; + + default: + goto loop_end; + } + } +loop_end: + /* malloc for temporary buffer, add 1 for '\0' */ + number_c_string = (unsigned char *) input_buffer->hooks.allocate(number_string_length + 1); + if (number_c_string == NULL) + { + return false; /* allocation failure */ + } + + memcpy(number_c_string, buffer_at_offset(input_buffer), number_string_length); + number_c_string[number_string_length] = '\0'; + + if (has_decimal_point) + { + for (i = 0; i < number_string_length; i++) + { + if (number_c_string[i] == '.') + { + /* replace '.' with the decimal point of the current locale (for strtod) */ + number_c_string[i] = decimal_point; + } + } + } + + number = strtod((const char*)number_c_string, (char**)&after_end); + if (number_c_string == after_end) + { + /* free the temporary buffer */ + input_buffer->hooks.deallocate(number_c_string); + return false; /* parse_error */ + } + + item->valuedouble = number; + + /* use saturation in case of overflow */ + if (number >= INT_MAX) + { + item->valueint = INT_MAX; + } + else if (number <= (double)INT_MIN) + { + item->valueint = INT_MIN; + } + else + { + item->valueint = (int)number; + } + + item->type = cJSON_Number; + + input_buffer->offset += (size_t)(after_end - number_c_string); + /* free the temporary buffer */ + input_buffer->hooks.deallocate(number_c_string); + return true; +} + +/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */ +CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) +{ + if (number >= INT_MAX) + { + object->valueint = INT_MAX; + } + else if (number <= (double)INT_MIN) + { + object->valueint = INT_MIN; + } + else + { + object->valueint = (int)number; + } + + return object->valuedouble = number; +} + +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as an error and return NULL */ +CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) +{ + char *copy = NULL; + size_t v1_len; + size_t v2_len; + /* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */ + if ((object == NULL) || !(object->type & cJSON_String) || (object->type & cJSON_IsReference)) + { + return NULL; + } + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) + { + return NULL; + } + + v1_len = strlen(valuestring); + v2_len = strlen(object->valuestring); + + if (v1_len <= v2_len) + { + /* strcpy does not handle overlapping string: [X1, X2] [Y1, Y2] => X2 < Y1 or Y2 < X1 */ + if (!( valuestring + v1_len < object->valuestring || object->valuestring + v2_len < valuestring )) + { + return NULL; + } + strcpy(object->valuestring, valuestring); + return object->valuestring; + } + copy = (char*) cJSON_strdup((const unsigned char*)valuestring, &global_hooks); + if (copy == NULL) + { + return NULL; + } + if (object->valuestring != NULL) + { + cJSON_free(object->valuestring); + } + object->valuestring = copy; + + return copy; +} + +typedef struct +{ + unsigned char *buffer; + size_t length; + size_t offset; + size_t depth; /* current nesting depth (for formatted printing) */ + cJSON_bool noalloc; + cJSON_bool format; /* is this print a formatted print */ + internal_hooks hooks; +} printbuffer; + +/* realloc printbuffer if necessary to have at least "needed" bytes more */ +static unsigned char* ensure(printbuffer * const p, size_t needed) +{ + unsigned char *newbuffer = NULL; + size_t newsize = 0; + + if ((p == NULL) || (p->buffer == NULL)) + { + return NULL; + } + + if ((p->length > 0) && (p->offset >= p->length)) + { + /* make sure that offset is valid */ + return NULL; + } + + if (needed > INT_MAX) + { + /* sizes bigger than INT_MAX are currently not supported */ + return NULL; + } + + needed += p->offset + 1; + if (needed <= p->length) + { + return p->buffer + p->offset; + } + + if (p->noalloc) { + return NULL; + } + + /* calculate new buffer size */ + if (needed > (INT_MAX / 2)) + { + /* overflow of int, use INT_MAX if possible */ + if (needed <= INT_MAX) + { + newsize = INT_MAX; + } + else + { + return NULL; + } + } + else + { + newsize = needed * 2; + } + + if (p->hooks.reallocate != NULL) + { + /* reallocate with realloc if available */ + newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize); + if (newbuffer == NULL) + { + p->hooks.deallocate(p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + } + else + { + /* otherwise reallocate manually */ + newbuffer = (unsigned char*)p->hooks.allocate(newsize); + if (!newbuffer) + { + p->hooks.deallocate(p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + + memcpy(newbuffer, p->buffer, p->offset + 1); + p->hooks.deallocate(p->buffer); + } + p->length = newsize; + p->buffer = newbuffer; + + return newbuffer + p->offset; +} + +/* calculate the new length of the string in a printbuffer and update the offset */ +static void update_offset(printbuffer * const buffer) +{ + const unsigned char *buffer_pointer = NULL; + if ((buffer == NULL) || (buffer->buffer == NULL)) + { + return; + } + buffer_pointer = buffer->buffer + buffer->offset; + + buffer->offset += strlen((const char*)buffer_pointer); +} + +/* securely comparison of floating-point variables */ +static cJSON_bool compare_double(double a, double b) +{ + double maxVal = fabs(a) > fabs(b) ? fabs(a) : fabs(b); + return (fabs(a - b) <= maxVal * DBL_EPSILON); +} + +/* Render the number nicely from the given item into a string. */ +static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + double d = item->valuedouble; + int length = 0; + size_t i = 0; + unsigned char number_buffer[26] = {0}; /* temporary buffer to print the number into */ + unsigned char decimal_point = get_decimal_point(); + double test = 0.0; + + if (output_buffer == NULL) + { + return false; + } + + /* This checks for NaN and Infinity */ + if (isnan(d) || isinf(d)) + { + length = sprintf((char*)number_buffer, "null"); + } + else if(compare_double(d, (double)item->valueint)) + { + length = sprintf((char*)number_buffer, "%d", item->valueint); + } + else + { + /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ + length = sprintf((char*)number_buffer, "%1.15g", d); + + /* Check whether the original double can be recovered */ + if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || !compare_double((double)test, d)) + { + /* If not, print with 17 decimal places of precision */ + length = sprintf((char*)number_buffer, "%1.17g", d); + } + } + + /* sprintf failed or buffer overrun occurred */ + if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) + { + return false; + } + + /* reserve appropriate space in the output */ + output_pointer = ensure(output_buffer, (size_t)length + sizeof("")); + if (output_pointer == NULL) + { + return false; + } + + /* copy the printed number to the output and replace locale + * dependent decimal point with '.' */ + for (i = 0; i < ((size_t)length); i++) + { + if (number_buffer[i] == decimal_point) + { + output_pointer[i] = '.'; + continue; + } + + output_pointer[i] = number_buffer[i]; + } + output_pointer[i] = '\0'; + + output_buffer->offset += (size_t)length; + + return true; +} + +/* parse 4 digit hexadecimal number */ +static unsigned parse_hex4(const unsigned char * const input) +{ + unsigned int h = 0; + size_t i = 0; + + for (i = 0; i < 4; i++) + { + /* parse digit */ + if ((input[i] >= '0') && (input[i] <= '9')) + { + h += (unsigned int) input[i] - '0'; + } + else if ((input[i] >= 'A') && (input[i] <= 'F')) + { + h += (unsigned int) 10 + input[i] - 'A'; + } + else if ((input[i] >= 'a') && (input[i] <= 'f')) + { + h += (unsigned int) 10 + input[i] - 'a'; + } + else /* invalid */ + { + return 0; + } + + if (i < 3) + { + /* shift left to make place for the next nibble */ + h = h << 4; + } + } + + return h; +} + +/* converts a UTF-16 literal to UTF-8 + * A literal can be one or two sequences of the form \uXXXX */ +static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) +{ + long unsigned int codepoint = 0; + unsigned int first_code = 0; + const unsigned char *first_sequence = input_pointer; + unsigned char utf8_length = 0; + unsigned char utf8_position = 0; + unsigned char sequence_length = 0; + unsigned char first_byte_mark = 0; + + if ((input_end - first_sequence) < 6) + { + /* input ends unexpectedly */ + goto fail; + } + + /* get the first utf16 sequence */ + first_code = parse_hex4(first_sequence + 2); + + /* check that the code is valid */ + if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) + { + goto fail; + } + + /* UTF16 surrogate pair */ + if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) + { + const unsigned char *second_sequence = first_sequence + 6; + unsigned int second_code = 0; + sequence_length = 12; /* \uXXXX\uXXXX */ + + if ((input_end - second_sequence) < 6) + { + /* input ends unexpectedly */ + goto fail; + } + + if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u')) + { + /* missing second half of the surrogate pair */ + goto fail; + } + + /* get the second utf16 sequence */ + second_code = parse_hex4(second_sequence + 2); + /* check that the code is valid */ + if ((second_code < 0xDC00) || (second_code > 0xDFFF)) + { + /* invalid second half of the surrogate pair */ + goto fail; + } + + + /* calculate the unicode codepoint from the surrogate pair */ + codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF)); + } + else + { + sequence_length = 6; /* \uXXXX */ + codepoint = first_code; + } + + /* encode as UTF-8 + * takes at maximum 4 bytes to encode: + * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + if (codepoint < 0x80) + { + /* normal ascii, encoding 0xxxxxxx */ + utf8_length = 1; + } + else if (codepoint < 0x800) + { + /* two bytes, encoding 110xxxxx 10xxxxxx */ + utf8_length = 2; + first_byte_mark = 0xC0; /* 11000000 */ + } + else if (codepoint < 0x10000) + { + /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ + utf8_length = 3; + first_byte_mark = 0xE0; /* 11100000 */ + } + else if (codepoint <= 0x10FFFF) + { + /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ + utf8_length = 4; + first_byte_mark = 0xF0; /* 11110000 */ + } + else + { + /* invalid unicode codepoint */ + goto fail; + } + + /* encode as utf8 */ + for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--) + { + /* 10xxxxxx */ + (*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF); + codepoint >>= 6; + } + /* encode first byte */ + if (utf8_length > 1) + { + (*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF); + } + else + { + (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); + } + + *output_pointer += utf8_length; + + return sequence_length; + +fail: + return 0; +} + +/* Parse the input text into an unescaped cinput, and populate item. */ +static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer) +{ + const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; + const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; + unsigned char *output_pointer = NULL; + unsigned char *output = NULL; + + /* not a string */ + if (buffer_at_offset(input_buffer)[0] != '\"') + { + goto fail; + } + + { + /* calculate approximate size of the output (overestimate) */ + size_t allocation_length = 0; + size_t skipped_bytes = 0; + while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"')) + { + /* is escape sequence */ + if (input_end[0] == '\\') + { + if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length) + { + /* prevent buffer overflow when last input character is a backslash */ + goto fail; + } + skipped_bytes++; + input_end++; + } + input_end++; + } + if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"')) + { + goto fail; /* string ended unexpectedly */ + } + + /* This is at most how much we need for the output */ + allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes; + output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof("")); + if (output == NULL) + { + goto fail; /* allocation failure */ + } + } + + output_pointer = output; + /* loop through the string literal */ + while (input_pointer < input_end) + { + if (*input_pointer != '\\') + { + *output_pointer++ = *input_pointer++; + } + /* escape sequence */ + else + { + unsigned char sequence_length = 2; + if ((input_end - input_pointer) < 1) + { + goto fail; + } + + switch (input_pointer[1]) + { + case 'b': + *output_pointer++ = '\b'; + break; + case 'f': + *output_pointer++ = '\f'; + break; + case 'n': + *output_pointer++ = '\n'; + break; + case 'r': + *output_pointer++ = '\r'; + break; + case 't': + *output_pointer++ = '\t'; + break; + case '\"': + case '\\': + case '/': + *output_pointer++ = input_pointer[1]; + break; + + /* UTF-16 literal */ + case 'u': + sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer); + if (sequence_length == 0) + { + /* failed to convert UTF16-literal to UTF-8 */ + goto fail; + } + break; + + default: + goto fail; + } + input_pointer += sequence_length; + } + } + + /* zero terminate the output */ + *output_pointer = '\0'; + + item->type = cJSON_String; + item->valuestring = (char*)output; + + input_buffer->offset = (size_t) (input_end - input_buffer->content); + input_buffer->offset++; + + return true; + +fail: + if (output != NULL) + { + input_buffer->hooks.deallocate(output); + output = NULL; + } + + if (input_pointer != NULL) + { + input_buffer->offset = (size_t)(input_pointer - input_buffer->content); + } + + return false; +} + +/* Render the cstring provided to an escaped version that can be printed. */ +static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer) +{ + const unsigned char *input_pointer = NULL; + unsigned char *output = NULL; + unsigned char *output_pointer = NULL; + size_t output_length = 0; + /* numbers of additional characters needed for escaping */ + size_t escape_characters = 0; + + if (output_buffer == NULL) + { + return false; + } + + /* empty string */ + if (input == NULL) + { + output = ensure(output_buffer, sizeof("\"\"")); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "\"\""); + + return true; + } + + /* set "flag" to 1 if something needs to be escaped */ + for (input_pointer = input; *input_pointer; input_pointer++) + { + switch (*input_pointer) + { + case '\"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + /* one character escape sequence */ + escape_characters++; + break; + default: + if (*input_pointer < 32) + { + /* UTF-16 escape sequence uXXXX */ + escape_characters += 5; + } + break; + } + } + output_length = (size_t)(input_pointer - input) + escape_characters; + + output = ensure(output_buffer, output_length + sizeof("\"\"")); + if (output == NULL) + { + return false; + } + + /* no characters have to be escaped */ + if (escape_characters == 0) + { + output[0] = '\"'; + memcpy(output + 1, input, output_length); + output[output_length + 1] = '\"'; + output[output_length + 2] = '\0'; + + return true; + } + + output[0] = '\"'; + output_pointer = output + 1; + /* copy the string */ + for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++) + { + if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\')) + { + /* normal character, copy */ + *output_pointer = *input_pointer; + } + else + { + /* character needs to be escaped */ + *output_pointer++ = '\\'; + switch (*input_pointer) + { + case '\\': + *output_pointer = '\\'; + break; + case '\"': + *output_pointer = '\"'; + break; + case '\b': + *output_pointer = 'b'; + break; + case '\f': + *output_pointer = 'f'; + break; + case '\n': + *output_pointer = 'n'; + break; + case '\r': + *output_pointer = 'r'; + break; + case '\t': + *output_pointer = 't'; + break; + default: + /* escape and print as unicode codepoint */ + sprintf((char*)output_pointer, "u%04x", *input_pointer); + output_pointer += 4; + break; + } + } + } + output[output_length + 1] = '\"'; + output[output_length + 2] = '\0'; + + return true; +} + +/* Invoke print_string_ptr (which is useful) on an item. */ +static cJSON_bool print_string(const cJSON * const item, printbuffer * const p) +{ + return print_string_ptr((unsigned char*)item->valuestring, p); +} + +/* Predeclare these prototypes. */ +static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer); +static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer); +static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer); + +/* Utility to jump whitespace and cr/lf */ +static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer) +{ + if ((buffer == NULL) || (buffer->content == NULL)) + { + return NULL; + } + + if (cannot_access_at_index(buffer, 0)) + { + return buffer; + } + + while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32)) + { + buffer->offset++; + } + + if (buffer->offset == buffer->length) + { + buffer->offset--; + } + + return buffer; +} + +/* skip the UTF-8 BOM (byte order mark) if it is at the beginning of a buffer */ +static parse_buffer *skip_utf8_bom(parse_buffer * const buffer) +{ + if ((buffer == NULL) || (buffer->content == NULL) || (buffer->offset != 0)) + { + return NULL; + } + + if (can_access_at_index(buffer, 4) && (strncmp((const char*)buffer_at_offset(buffer), "\xEF\xBB\xBF", 3) == 0)) + { + buffer->offset += 3; + } + + return buffer; +} + +CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated) +{ + size_t buffer_length; + + if (NULL == value) + { + return NULL; + } + + /* Adding null character size due to require_null_terminated. */ + buffer_length = strlen(value) + sizeof(""); + + return cJSON_ParseWithLengthOpts(value, buffer_length, return_parse_end, require_null_terminated); +} + +/* Parse an object - create a new root, and populate. */ +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated) +{ + parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } }; + cJSON *item = NULL; + + /* reset error position */ + global_error.json = NULL; + global_error.position = 0; + + if (value == NULL || 0 == buffer_length) + { + goto fail; + } + + buffer.content = (const unsigned char*)value; + buffer.length = buffer_length; + buffer.offset = 0; + buffer.hooks = global_hooks; + + item = cJSON_New_Item(&global_hooks); + if (item == NULL) /* memory fail */ + { + goto fail; + } + + if (!parse_value(item, buffer_skip_whitespace(skip_utf8_bom(&buffer)))) + { + /* parse failure. ep is set. */ + goto fail; + } + + /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ + if (require_null_terminated) + { + buffer_skip_whitespace(&buffer); + if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0') + { + goto fail; + } + } + if (return_parse_end) + { + *return_parse_end = (const char*)buffer_at_offset(&buffer); + } + + return item; + +fail: + if (item != NULL) + { + cJSON_Delete(item); + } + + if (value != NULL) + { + error local_error; + local_error.json = (const unsigned char*)value; + local_error.position = 0; + + if (buffer.offset < buffer.length) + { + local_error.position = buffer.offset; + } + else if (buffer.length > 0) + { + local_error.position = buffer.length - 1; + } + + if (return_parse_end != NULL) + { + *return_parse_end = (const char*)local_error.json + local_error.position; + } + + global_error = local_error; + } + + return NULL; +} + +/* Default options for cJSON_Parse */ +CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) +{ + return cJSON_ParseWithOpts(value, 0, 0); +} + +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLength(const char *value, size_t buffer_length) +{ + return cJSON_ParseWithLengthOpts(value, buffer_length, 0, 0); +} + +#define cjson_min(a, b) (((a) < (b)) ? (a) : (b)) + +static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks) +{ + static const size_t default_buffer_size = 256; + printbuffer buffer[1]; + unsigned char *printed = NULL; + + memset(buffer, 0, sizeof(buffer)); + + /* create buffer */ + buffer->buffer = (unsigned char*) hooks->allocate(default_buffer_size); + buffer->length = default_buffer_size; + buffer->format = format; + buffer->hooks = *hooks; + if (buffer->buffer == NULL) + { + goto fail; + } + + /* print the value */ + if (!print_value(item, buffer)) + { + goto fail; + } + update_offset(buffer); + + /* check if reallocate is available */ + if (hooks->reallocate != NULL) + { + printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->offset + 1); + if (printed == NULL) { + goto fail; + } + buffer->buffer = NULL; + } + else /* otherwise copy the JSON over to a new buffer */ + { + printed = (unsigned char*) hooks->allocate(buffer->offset + 1); + if (printed == NULL) + { + goto fail; + } + memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1)); + printed[buffer->offset] = '\0'; /* just to be sure */ + + /* free the buffer */ + hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; + } + + return printed; + +fail: + if (buffer->buffer != NULL) + { + hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; + } + + if (printed != NULL) + { + hooks->deallocate(printed); + printed = NULL; + } + + return NULL; +} + +/* Render a cJSON item/entity/structure to text. */ +CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) +{ + return (char*)print(item, true, &global_hooks); +} + +CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) +{ + return (char*)print(item, false, &global_hooks); +} + +CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) +{ + printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; + + if (prebuffer < 0) + { + return NULL; + } + + p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer); + if (!p.buffer) + { + return NULL; + } + + p.length = (size_t)prebuffer; + p.offset = 0; + p.noalloc = false; + p.format = fmt; + p.hooks = global_hooks; + + if (!print_value(item, &p)) + { + global_hooks.deallocate(p.buffer); + p.buffer = NULL; + return NULL; + } + + return (char*)p.buffer; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format) +{ + printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; + + if ((length < 0) || (buffer == NULL)) + { + return false; + } + + p.buffer = (unsigned char*)buffer; + p.length = (size_t)length; + p.offset = 0; + p.noalloc = true; + p.format = format; + p.hooks = global_hooks; + + return print_value(item, &p); +} + +/* Parser core - when encountering text, process appropriately. */ +static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer) +{ + if ((input_buffer == NULL) || (input_buffer->content == NULL)) + { + return false; /* no input */ + } + + /* parse the different types of values */ + /* null */ + if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0)) + { + item->type = cJSON_NULL; + input_buffer->offset += 4; + return true; + } + /* false */ + if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0)) + { + item->type = cJSON_False; + input_buffer->offset += 5; + return true; + } + /* true */ + if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0)) + { + item->type = cJSON_True; + item->valueint = 1; + input_buffer->offset += 4; + return true; + } + /* string */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"')) + { + return parse_string(item, input_buffer); + } + /* number */ + if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9')))) + { + return parse_number(item, input_buffer); + } + /* array */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '[')) + { + return parse_array(item, input_buffer); + } + /* object */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{')) + { + return parse_object(item, input_buffer); + } + + return false; +} + +/* Render a value to text. */ +static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output = NULL; + + if ((item == NULL) || (output_buffer == NULL)) + { + return false; + } + + switch ((item->type) & 0xFF) + { + case cJSON_NULL: + output = ensure(output_buffer, 5); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "null"); + return true; + + case cJSON_False: + output = ensure(output_buffer, 6); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "false"); + return true; + + case cJSON_True: + output = ensure(output_buffer, 5); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "true"); + return true; + + case cJSON_Number: + return print_number(item, output_buffer); + + case cJSON_Raw: + { + size_t raw_length = 0; + if (item->valuestring == NULL) + { + return false; + } + + raw_length = strlen(item->valuestring) + sizeof(""); + output = ensure(output_buffer, raw_length); + if (output == NULL) + { + return false; + } + memcpy(output, item->valuestring, raw_length); + return true; + } + + case cJSON_String: + return print_string(item, output_buffer); + + case cJSON_Array: + return print_array(item, output_buffer); + + case cJSON_Object: + return print_object(item, output_buffer); + + default: + return false; + } +} + +/* Build an array from input text. */ +static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer) +{ + cJSON *head = NULL; /* head of the linked list */ + cJSON *current_item = NULL; + + if (input_buffer->depth >= CJSON_NESTING_LIMIT) + { + return false; /* to deeply nested */ + } + input_buffer->depth++; + + if (buffer_at_offset(input_buffer)[0] != '[') + { + /* not an array */ + goto fail; + } + + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']')) + { + /* empty array */ + goto success; + } + + /* check if we skipped to the end of the buffer */ + if (cannot_access_at_index(input_buffer, 0)) + { + input_buffer->offset--; + goto fail; + } + + /* step back to character in front of the first element */ + input_buffer->offset--; + /* loop through the comma separated array elements */ + do + { + /* allocate next item */ + cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); + if (new_item == NULL) + { + goto fail; /* allocation failure */ + } + + /* attach next item to list */ + if (head == NULL) + { + /* start the linked list */ + current_item = head = new_item; + } + else + { + /* add to the end and advance */ + current_item->next = new_item; + new_item->prev = current_item; + current_item = new_item; + } + + /* parse next value */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_value(current_item, input_buffer)) + { + goto fail; /* failed to parse value */ + } + buffer_skip_whitespace(input_buffer); + } + while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); + + if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']') + { + goto fail; /* expected end of array */ + } + +success: + input_buffer->depth--; + + if (head != NULL) { + head->prev = current_item; + } + + item->type = cJSON_Array; + item->child = head; + + input_buffer->offset++; + + return true; + +fail: + if (head != NULL) + { + cJSON_Delete(head); + } + + return false; +} + +/* Render an array to text */ +static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + size_t length = 0; + cJSON *current_element = item->child; + + if (output_buffer == NULL) + { + return false; + } + + /* Compose the output array. */ + /* opening square bracket */ + output_pointer = ensure(output_buffer, 1); + if (output_pointer == NULL) + { + return false; + } + + *output_pointer = '['; + output_buffer->offset++; + output_buffer->depth++; + + while (current_element != NULL) + { + if (!print_value(current_element, output_buffer)) + { + return false; + } + update_offset(output_buffer); + if (current_element->next) + { + length = (size_t) (output_buffer->format ? 2 : 1); + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ','; + if(output_buffer->format) + { + *output_pointer++ = ' '; + } + *output_pointer = '\0'; + output_buffer->offset += length; + } + current_element = current_element->next; + } + + output_pointer = ensure(output_buffer, 2); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ']'; + *output_pointer = '\0'; + output_buffer->depth--; + + return true; +} + +/* Build an object from the text. */ +static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer) +{ + cJSON *head = NULL; /* linked list head */ + cJSON *current_item = NULL; + + if (input_buffer->depth >= CJSON_NESTING_LIMIT) + { + return false; /* to deeply nested */ + } + input_buffer->depth++; + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{')) + { + goto fail; /* not an object */ + } + + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}')) + { + goto success; /* empty object */ + } + + /* check if we skipped to the end of the buffer */ + if (cannot_access_at_index(input_buffer, 0)) + { + input_buffer->offset--; + goto fail; + } + + /* step back to character in front of the first element */ + input_buffer->offset--; + /* loop through the comma separated array elements */ + do + { + /* allocate next item */ + cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); + if (new_item == NULL) + { + goto fail; /* allocation failure */ + } + + /* attach next item to list */ + if (head == NULL) + { + /* start the linked list */ + current_item = head = new_item; + } + else + { + /* add to the end and advance */ + current_item->next = new_item; + new_item->prev = current_item; + current_item = new_item; + } + + if (cannot_access_at_index(input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + + /* parse the name of the child */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_string(current_item, input_buffer)) + { + goto fail; /* failed to parse name */ + } + buffer_skip_whitespace(input_buffer); + + /* swap valuestring and string, because we parsed the name */ + current_item->string = current_item->valuestring; + current_item->valuestring = NULL; + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':')) + { + goto fail; /* invalid object */ + } + + /* parse the value */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_value(current_item, input_buffer)) + { + goto fail; /* failed to parse value */ + } + buffer_skip_whitespace(input_buffer); + } + while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}')) + { + goto fail; /* expected end of object */ + } + +success: + input_buffer->depth--; + + if (head != NULL) { + head->prev = current_item; + } + + item->type = cJSON_Object; + item->child = head; + + input_buffer->offset++; + return true; + +fail: + if (head != NULL) + { + cJSON_Delete(head); + } + + return false; +} + +/* Render an object to text. */ +static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + size_t length = 0; + cJSON *current_item = item->child; + + if (output_buffer == NULL) + { + return false; + } + + /* Compose the output: */ + length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */ + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) + { + return false; + } + + *output_pointer++ = '{'; + output_buffer->depth++; + if (output_buffer->format) + { + *output_pointer++ = '\n'; + } + output_buffer->offset += length; + + while (current_item) + { + if (output_buffer->format) + { + size_t i; + output_pointer = ensure(output_buffer, output_buffer->depth); + if (output_pointer == NULL) + { + return false; + } + for (i = 0; i < output_buffer->depth; i++) + { + *output_pointer++ = '\t'; + } + output_buffer->offset += output_buffer->depth; + } + + /* print key */ + if (!print_string_ptr((unsigned char*)current_item->string, output_buffer)) + { + return false; + } + update_offset(output_buffer); + + length = (size_t) (output_buffer->format ? 2 : 1); + output_pointer = ensure(output_buffer, length); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ':'; + if (output_buffer->format) + { + *output_pointer++ = '\t'; + } + output_buffer->offset += length; + + /* print value */ + if (!print_value(current_item, output_buffer)) + { + return false; + } + update_offset(output_buffer); + + /* print comma if not last */ + length = ((size_t)(output_buffer->format ? 1 : 0) + (size_t)(current_item->next ? 1 : 0)); + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) + { + return false; + } + if (current_item->next) + { + *output_pointer++ = ','; + } + + if (output_buffer->format) + { + *output_pointer++ = '\n'; + } + *output_pointer = '\0'; + output_buffer->offset += length; + + current_item = current_item->next; + } + + output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2); + if (output_pointer == NULL) + { + return false; + } + if (output_buffer->format) + { + size_t i; + for (i = 0; i < (output_buffer->depth - 1); i++) + { + *output_pointer++ = '\t'; + } + } + *output_pointer++ = '}'; + *output_pointer = '\0'; + output_buffer->depth--; + + return true; +} + +/* Get Array size/item / object item. */ +CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) +{ + cJSON *child = NULL; + size_t size = 0; + + if (array == NULL) + { + return 0; + } + + child = array->child; + + while(child != NULL) + { + size++; + child = child->next; + } + + /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ + + return (int)size; +} + +static cJSON* get_array_item(const cJSON *array, size_t index) +{ + cJSON *current_child = NULL; + + if (array == NULL) + { + return NULL; + } + + current_child = array->child; + while ((current_child != NULL) && (index > 0)) + { + index--; + current_child = current_child->next; + } + + return current_child; +} + +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) +{ + if (index < 0) + { + return NULL; + } + + return get_array_item(array, (size_t)index); +} + +static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) +{ + cJSON *current_element = NULL; + + if ((object == NULL) || (name == NULL)) + { + return NULL; + } + + current_element = object->child; + if (case_sensitive) + { + while ((current_element != NULL) && (current_element->string != NULL) && (strcmp(name, current_element->string) != 0)) + { + current_element = current_element->next; + } + } + else + { + while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0)) + { + current_element = current_element->next; + } + } + + if ((current_element == NULL) || (current_element->string == NULL)) { + return NULL; + } + + return current_element; +} + +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string) +{ + return get_object_item(object, string, false); +} + +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string) +{ + return get_object_item(object, string, true); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string) +{ + return cJSON_GetObjectItem(object, string) ? 1 : 0; +} + +/* Utility for array list handling. */ +static void suffix_object(cJSON *prev, cJSON *item) +{ + prev->next = item; + item->prev = prev; +} + +/* Utility for handling references. */ +static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks) +{ + cJSON *reference = NULL; + if (item == NULL) + { + return NULL; + } + + reference = cJSON_New_Item(hooks); + if (reference == NULL) + { + return NULL; + } + + memcpy(reference, item, sizeof(cJSON)); + reference->string = NULL; + reference->type |= cJSON_IsReference; + reference->next = reference->prev = NULL; + return reference; +} + +static cJSON_bool add_item_to_array(cJSON *array, cJSON *item) +{ + cJSON *child = NULL; + + if ((item == NULL) || (array == NULL) || (array == item)) + { + return false; + } + + child = array->child; + /* + * To find the last item in array quickly, we use prev in array + */ + if (child == NULL) + { + /* list is empty, start new one */ + array->child = item; + item->prev = item; + item->next = NULL; + } + else + { + /* append to the end */ + if (child->prev) + { + suffix_object(child->prev, item); + array->child->prev = item; + } + } + + return true; +} + +/* Add item to array/object. */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item) +{ + return add_item_to_array(array, item); +} + +#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) + #pragma GCC diagnostic push +#endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif +/* helper function to cast away const */ +static void* cast_away_const(const void* string) +{ + return (void*)string; +} +#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) + #pragma GCC diagnostic pop +#endif + + +static cJSON_bool add_item_to_object(cJSON * const object, const char * const string, cJSON * const item, const internal_hooks * const hooks, const cJSON_bool constant_key) +{ + char *new_key = NULL; + int new_type = cJSON_Invalid; + + if ((object == NULL) || (string == NULL) || (item == NULL) || (object == item)) + { + return false; + } + + if (constant_key) + { + new_key = (char*)cast_away_const(string); + new_type = item->type | cJSON_StringIsConst; + } + else + { + new_key = (char*)cJSON_strdup((const unsigned char*)string, hooks); + if (new_key == NULL) + { + return false; + } + + new_type = item->type & ~cJSON_StringIsConst; + } + + if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) + { + hooks->deallocate(item->string); + } + + item->string = new_key; + item->type = new_type; + + return add_item_to_array(object, item); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) +{ + return add_item_to_object(object, string, item, &global_hooks, false); +} + +/* Add an item to an object with constant string as key */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) +{ + return add_item_to_object(object, string, item, &global_hooks, true); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) +{ + if (array == NULL) + { + return false; + } + + return add_item_to_array(array, create_reference(item, &global_hooks)); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) +{ + if ((object == NULL) || (string == NULL)) + { + return false; + } + + return add_item_to_object(object, string, create_reference(item, &global_hooks), &global_hooks, false); +} + +CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name) +{ + cJSON *null = cJSON_CreateNull(); + if (add_item_to_object(object, name, null, &global_hooks, false)) + { + return null; + } + + cJSON_Delete(null); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name) +{ + cJSON *true_item = cJSON_CreateTrue(); + if (add_item_to_object(object, name, true_item, &global_hooks, false)) + { + return true_item; + } + + cJSON_Delete(true_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name) +{ + cJSON *false_item = cJSON_CreateFalse(); + if (add_item_to_object(object, name, false_item, &global_hooks, false)) + { + return false_item; + } + + cJSON_Delete(false_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean) +{ + cJSON *bool_item = cJSON_CreateBool(boolean); + if (add_item_to_object(object, name, bool_item, &global_hooks, false)) + { + return bool_item; + } + + cJSON_Delete(bool_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number) +{ + cJSON *number_item = cJSON_CreateNumber(number); + if (add_item_to_object(object, name, number_item, &global_hooks, false)) + { + return number_item; + } + + cJSON_Delete(number_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string) +{ + cJSON *string_item = cJSON_CreateString(string); + if (add_item_to_object(object, name, string_item, &global_hooks, false)) + { + return string_item; + } + + cJSON_Delete(string_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw) +{ + cJSON *raw_item = cJSON_CreateRaw(raw); + if (add_item_to_object(object, name, raw_item, &global_hooks, false)) + { + return raw_item; + } + + cJSON_Delete(raw_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name) +{ + cJSON *object_item = cJSON_CreateObject(); + if (add_item_to_object(object, name, object_item, &global_hooks, false)) + { + return object_item; + } + + cJSON_Delete(object_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name) +{ + cJSON *array = cJSON_CreateArray(); + if (add_item_to_object(object, name, array, &global_hooks, false)) + { + return array; + } + + cJSON_Delete(array); + return NULL; +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item) +{ + if ((parent == NULL) || (item == NULL) || (item != parent->child && item->prev == NULL)) + { + return NULL; + } + + if (item != parent->child) + { + /* not the first element */ + item->prev->next = item->next; + } + if (item->next != NULL) + { + /* not the last element */ + item->next->prev = item->prev; + } + + if (item == parent->child) + { + /* first element */ + parent->child = item->next; + } + else if (item->next == NULL) + { + /* last element */ + parent->child->prev = item->prev; + } + + /* make sure the detached item doesn't point anywhere anymore */ + item->prev = NULL; + item->next = NULL; + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) +{ + if (which < 0) + { + return NULL; + } + + return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which)); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) +{ + cJSON_Delete(cJSON_DetachItemFromArray(array, which)); +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string) +{ + cJSON *to_detach = cJSON_GetObjectItem(object, string); + + return cJSON_DetachItemViaPointer(object, to_detach); +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) +{ + cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); + + return cJSON_DetachItemViaPointer(object, to_detach); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string) +{ + cJSON_Delete(cJSON_DetachItemFromObject(object, string)); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) +{ + cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); +} + +/* Replace array/object items with new ones. */ +CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) +{ + cJSON *after_inserted = NULL; + + if (which < 0 || newitem == NULL) + { + return false; + } + + after_inserted = get_array_item(array, (size_t)which); + if (after_inserted == NULL) + { + return add_item_to_array(array, newitem); + } + + if (after_inserted != array->child && after_inserted->prev == NULL) { + /* return false if after_inserted is a corrupted array item */ + return false; + } + + newitem->next = after_inserted; + newitem->prev = after_inserted->prev; + after_inserted->prev = newitem; + if (after_inserted == array->child) + { + array->child = newitem; + } + else + { + newitem->prev->next = newitem; + } + return true; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) +{ + if ((parent == NULL) || (parent->child == NULL) || (replacement == NULL) || (item == NULL)) + { + return false; + } + + if (replacement == item) + { + return true; + } + + replacement->next = item->next; + replacement->prev = item->prev; + + if (replacement->next != NULL) + { + replacement->next->prev = replacement; + } + if (parent->child == item) + { + if (parent->child->prev == parent->child) + { + replacement->prev = replacement; + } + parent->child = replacement; + } + else + { /* + * To find the last item in array quickly, we use prev in array. + * We can't modify the last item's next pointer where this item was the parent's child + */ + if (replacement->prev != NULL) + { + replacement->prev->next = replacement; + } + if (replacement->next == NULL) + { + parent->child->prev = replacement; + } + } + + item->next = NULL; + item->prev = NULL; + cJSON_Delete(item); + + return true; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) +{ + if (which < 0) + { + return false; + } + + return cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem); +} + +static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive) +{ + if ((replacement == NULL) || (string == NULL)) + { + return false; + } + + /* replace the name in the replacement */ + if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL)) + { + cJSON_free(replacement->string); + } + replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if (replacement->string == NULL) + { + return false; + } + + replacement->type &= ~cJSON_StringIsConst; + + return cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) +{ + return replace_item_in_object(object, string, newitem, false); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem) +{ + return replace_item_in_object(object, string, newitem, true); +} + +/* Create basic types: */ +CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_NULL; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_True; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_False; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = boolean ? cJSON_True : cJSON_False; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_Number; + item->valuedouble = num; + + /* use saturation in case of overflow */ + if (num >= INT_MAX) + { + item->valueint = INT_MAX; + } + else if (num <= (double)INT_MIN) + { + item->valueint = INT_MIN; + } + else + { + item->valueint = (int)num; + } + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_String; + item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if(!item->valuestring) + { + cJSON_Delete(item); + return NULL; + } + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) + { + item->type = cJSON_String | cJSON_IsReference; + item->valuestring = (char*)cast_away_const(string); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) { + item->type = cJSON_Object | cJSON_IsReference; + item->child = (cJSON*)cast_away_const(child); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child) { + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) { + item->type = cJSON_Array | cJSON_IsReference; + item->child = (cJSON*)cast_away_const(child); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_Raw; + item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks); + if(!item->valuestring) + { + cJSON_Delete(item); + return NULL; + } + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type=cJSON_Array; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item) + { + item->type = cJSON_Object; + } + + return item; +} + +/* Create Arrays: */ +CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber(numbers[i]); + if (!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p, n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber((double)numbers[i]); + if(!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p, n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber(numbers[i]); + if(!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p, n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (strings == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for (i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateString(strings[i]); + if(!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p,n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +/* Duplication */ +cJSON * cJSON_Duplicate_rec(const cJSON *item, size_t depth, cJSON_bool recurse); + +CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) +{ + return cJSON_Duplicate_rec(item, 0, recurse ); +} + +cJSON * cJSON_Duplicate_rec(const cJSON *item, size_t depth, cJSON_bool recurse) +{ + cJSON *newitem = NULL; + cJSON *child = NULL; + cJSON *next = NULL; + cJSON *newchild = NULL; + + /* Bail on bad ptr */ + if (!item) + { + goto fail; + } + /* Create new item */ + newitem = cJSON_New_Item(&global_hooks); + if (!newitem) + { + goto fail; + } + /* Copy over all vars */ + newitem->type = item->type & (~cJSON_IsReference); + newitem->valueint = item->valueint; + newitem->valuedouble = item->valuedouble; + if (item->valuestring) + { + newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks); + if (!newitem->valuestring) + { + goto fail; + } + } + if (item->string) + { + newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks); + if (!newitem->string) + { + goto fail; + } + } + /* If non-recursive, then we're done! */ + if (!recurse) + { + return newitem; + } + /* Walk the ->next chain for the child. */ + child = item->child; + while (child != NULL) + { + if(depth >= CJSON_CIRCULAR_LIMIT) { + goto fail; + } + newchild = cJSON_Duplicate_rec(child, depth + 1, true); /* Duplicate (with recurse) each item in the ->next chain */ + if (!newchild) + { + goto fail; + } + if (next != NULL) + { + /* If newitem->child already set, then crosswire ->prev and ->next and move on */ + next->next = newchild; + newchild->prev = next; + next = newchild; + } + else + { + /* Set newitem->child and move to it */ + newitem->child = newchild; + next = newchild; + } + child = child->next; + } + if (newitem && newitem->child) + { + newitem->child->prev = newchild; + } + + return newitem; + +fail: + if (newitem != NULL) + { + cJSON_Delete(newitem); + } + + return NULL; +} + +static void skip_oneline_comment(char **input) +{ + *input += static_strlen("//"); + + for (; (*input)[0] != '\0'; ++(*input)) + { + if ((*input)[0] == '\n') { + *input += static_strlen("\n"); + return; + } + } +} + +static void skip_multiline_comment(char **input) +{ + *input += static_strlen("/*"); + + for (; (*input)[0] != '\0'; ++(*input)) + { + if (((*input)[0] == '*') && ((*input)[1] == '/')) + { + *input += static_strlen("*/"); + return; + } + } +} + +static void minify_string(char **input, char **output) { + (*output)[0] = (*input)[0]; + *input += static_strlen("\""); + *output += static_strlen("\""); + + + for (; (*input)[0] != '\0'; (void)++(*input), ++(*output)) { + (*output)[0] = (*input)[0]; + + if ((*input)[0] == '\"') { + (*output)[0] = '\"'; + *input += static_strlen("\""); + *output += static_strlen("\""); + return; + } else if (((*input)[0] == '\\') && ((*input)[1] == '\"')) { + (*output)[1] = (*input)[1]; + *input += static_strlen("\""); + *output += static_strlen("\""); + } + } +} + +CJSON_PUBLIC(void) cJSON_Minify(char *json) +{ + char *into = json; + + if (json == NULL) + { + return; + } + + while (json[0] != '\0') + { + switch (json[0]) + { + case ' ': + case '\t': + case '\r': + case '\n': + json++; + break; + + case '/': + if (json[1] == '/') + { + skip_oneline_comment(&json); + } + else if (json[1] == '*') + { + skip_multiline_comment(&json); + } else { + json++; + } + break; + + case '\"': + minify_string(&json, (char**)&into); + break; + + default: + into[0] = json[0]; + json++; + into++; + } + } + + /* and null-terminate. */ + *into = '\0'; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Invalid; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_False; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xff) == cJSON_True; +} + + +CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & (cJSON_True | cJSON_False)) != 0; +} +CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_NULL; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Number; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_String; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Array; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Object; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Raw; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) +{ + if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF))) + { + return false; + } + + /* check if type is valid */ + switch (a->type & 0xFF) + { + case cJSON_False: + case cJSON_True: + case cJSON_NULL: + case cJSON_Number: + case cJSON_String: + case cJSON_Raw: + case cJSON_Array: + case cJSON_Object: + break; + + default: + return false; + } + + /* identical objects are equal */ + if (a == b) + { + return true; + } + + switch (a->type & 0xFF) + { + /* in these cases and equal type is enough */ + case cJSON_False: + case cJSON_True: + case cJSON_NULL: + return true; + + case cJSON_Number: + if (compare_double(a->valuedouble, b->valuedouble)) + { + return true; + } + return false; + + case cJSON_String: + case cJSON_Raw: + if ((a->valuestring == NULL) || (b->valuestring == NULL)) + { + return false; + } + if (strcmp(a->valuestring, b->valuestring) == 0) + { + return true; + } + + return false; + + case cJSON_Array: + { + cJSON *a_element = a->child; + cJSON *b_element = b->child; + + for (; (a_element != NULL) && (b_element != NULL);) + { + if (!cJSON_Compare(a_element, b_element, case_sensitive)) + { + return false; + } + + a_element = a_element->next; + b_element = b_element->next; + } + + /* one of the arrays is longer than the other */ + if (a_element != b_element) { + return false; + } + + return true; + } + + case cJSON_Object: + { + cJSON *a_element = NULL; + cJSON *b_element = NULL; + cJSON_ArrayForEach(a_element, a) + { + /* TODO This has O(n^2) runtime, which is horrible! */ + b_element = get_object_item(b, a_element->string, case_sensitive); + if (b_element == NULL) + { + return false; + } + + if (!cJSON_Compare(a_element, b_element, case_sensitive)) + { + return false; + } + } + + /* doing this twice, once on a and b to prevent true comparison if a subset of b + * TODO: Do this the proper way, this is just a fix for now */ + cJSON_ArrayForEach(b_element, b) + { + a_element = get_object_item(a, b_element->string, case_sensitive); + if (a_element == NULL) + { + return false; + } + + if (!cJSON_Compare(b_element, a_element, case_sensitive)) + { + return false; + } + } + + return true; + } + + default: + return false; + } +} + +CJSON_PUBLIC(void *) cJSON_malloc(size_t size) +{ + return global_hooks.allocate(size); +} + +CJSON_PUBLIC(void) cJSON_free(void *object) +{ + global_hooks.deallocate(object); + object = NULL; +} diff --git a/lib/librdkafka-2.15.0/src/cJSON.h b/lib/librdkafka-2.15.0/src/cJSON.h new file mode 100644 index 00000000000..cab5feb4277 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/cJSON.h @@ -0,0 +1,306 @@ +/* + Copyright (c) 2009-2017 Dave Gamble and cJSON contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#ifndef cJSON__h +#define cJSON__h + +#ifdef __cplusplus +extern "C" +{ +#endif + +#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) +#define __WINDOWS__ +#endif + +#ifdef __WINDOWS__ + +/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 3 define options: + +CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols +CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default) +CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol + +For *nix builds that support visibility attribute, you can define similar behavior by + +setting default visibility to hidden by adding +-fvisibility=hidden (for gcc) +or +-xldscope=hidden (for sun cc) +to CFLAGS + +then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does + +*/ + +#define CJSON_CDECL __cdecl +#define CJSON_STDCALL __stdcall + +/* export symbols by default, this is necessary for copy pasting the C and header file */ +#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS) +#define CJSON_EXPORT_SYMBOLS +#endif + +#if defined(CJSON_HIDE_SYMBOLS) +#define CJSON_PUBLIC(type) type CJSON_STDCALL +#elif defined(CJSON_EXPORT_SYMBOLS) +#define CJSON_PUBLIC(type) __declspec(dllexport) type CJSON_STDCALL +#elif defined(CJSON_IMPORT_SYMBOLS) +#define CJSON_PUBLIC(type) __declspec(dllimport) type CJSON_STDCALL +#endif +#else /* !__WINDOWS__ */ +#define CJSON_CDECL +#define CJSON_STDCALL + +#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY) +#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type +#else +#define CJSON_PUBLIC(type) type +#endif +#endif + +/* project version */ +#define CJSON_VERSION_MAJOR 1 +#define CJSON_VERSION_MINOR 7 +#define CJSON_VERSION_PATCH 19 + +#include + +/* cJSON Types: */ +#define cJSON_Invalid (0) +#define cJSON_False (1 << 0) +#define cJSON_True (1 << 1) +#define cJSON_NULL (1 << 2) +#define cJSON_Number (1 << 3) +#define cJSON_String (1 << 4) +#define cJSON_Array (1 << 5) +#define cJSON_Object (1 << 6) +#define cJSON_Raw (1 << 7) /* raw json */ + +#define cJSON_IsReference 256 +#define cJSON_StringIsConst 512 + +/* The cJSON structure: */ +typedef struct cJSON +{ + /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ + struct cJSON *next; + struct cJSON *prev; + /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ + struct cJSON *child; + + /* The type of the item, as above. */ + int type; + + /* The item's string, if type==cJSON_String and type == cJSON_Raw */ + char *valuestring; + /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */ + int valueint; + /* The item's number, if type==cJSON_Number */ + double valuedouble; + + /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ + char *string; +} cJSON; + +typedef struct cJSON_Hooks +{ + /* malloc/free are CDECL on Windows regardless of the default calling convention of the compiler, so ensure the hooks allow passing those functions directly. */ + void *(CJSON_CDECL *malloc_fn)(size_t sz); + void (CJSON_CDECL *free_fn)(void *ptr); +} cJSON_Hooks; + +typedef int cJSON_bool; + +/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them. + * This is to prevent stack overflows. */ +#ifndef CJSON_NESTING_LIMIT +#define CJSON_NESTING_LIMIT 1000 +#endif + +/* Limits the length of circular references can be before cJSON rejects to parse them. + * This is to prevent stack overflows. */ +#ifndef CJSON_CIRCULAR_LIMIT +#define CJSON_CIRCULAR_LIMIT 10000 +#endif + +/* returns the version of cJSON as a string */ +CJSON_PUBLIC(const char*) cJSON_Version(void); + +/* Supply malloc, realloc and free functions to cJSON */ +CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks); + +/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */ +/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */ +CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLength(const char *value, size_t buffer_length); +/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ +/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */ +CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated); +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated); + +/* Render a cJSON entity to text for transfer/storage. */ +CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); +/* Render a cJSON entity to text for transfer/storage without any formatting. */ +CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); +/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ +CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); +/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */ +/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */ +CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format); +/* Delete a cJSON entity and all subentities. */ +CJSON_PUBLIC(void) cJSON_Delete(cJSON *item); + +/* Returns the number of items in an array (or object). */ +CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); +/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */ +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); +/* Get item "string" from object. Case insensitive. */ +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string); +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string); +CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string); +/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ +CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); + +/* Check item type and return its value */ +CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item); +CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item); + +/* These functions check the type of an item */ +CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item); + +/* These calls create a cJSON item of the appropriate type. */ +CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); +CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); +CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); +/* raw json */ +CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); +CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); + +/* Create a string where valuestring references a string so + * it will not be freed by cJSON_Delete */ +CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string); +/* Create an object/array that only references it's elements so + * they will not be freed by cJSON_Delete */ +CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child); +CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child); + +/* These utilities create an Array of count items. + * The parameter count cannot be greater than the number of elements in the number array, otherwise array access will be out of bounds.*/ +CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int count); + +/* Append item to the specified array/object. */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item); +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); +/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object. + * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before + * writing to `item->string` */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); +/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); + +/* Remove/Detach items from Arrays/Objects. */ +CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); +CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); +CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); + +/* Update array items. */ +CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement); +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem); + +/* Duplicate a cJSON item */ +CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); +/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will + * need to be released. With recurse!=0, it will duplicate any children connected to the item. + * The item->next and ->prev pointers are always zero on return from Duplicate. */ +/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal. + * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ +CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); + +/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') from strings. + * The input pointer json cannot point to a read-only address area, such as a string constant, + * but should point to a readable and writable address area. */ +CJSON_PUBLIC(void) cJSON_Minify(char *json); + +/* Helper functions for creating and adding items to an object at the same time. + * They return the added item or NULL on failure. */ +CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean); +CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number); +CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string); +CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw); +CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name); + +/* When assigning an integer value, it needs to be propagated to valuedouble too. */ +#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number)) +/* helper for the cJSON_SetNumberValue macro */ +CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); +#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number)) +/* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */ +CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring); + +/* If the object is not a boolean type this does nothing and returns cJSON_Invalid else it returns the new type*/ +#define cJSON_SetBoolValue(object, boolValue) ( \ + (object != NULL && ((object)->type & (cJSON_False|cJSON_True))) ? \ + (object)->type=((object)->type &(~(cJSON_False|cJSON_True)))|((boolValue)?cJSON_True:cJSON_False) : \ + cJSON_Invalid\ +) + +/* Macro for iterating over an array or object */ +#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) + +/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ +CJSON_PUBLIC(void *) cJSON_malloc(size_t size); +CJSON_PUBLIC(void) cJSON_free(void *object); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/librdkafka-2.10.1/src/crc32c.c b/lib/librdkafka-2.15.0/src/crc32c.c similarity index 100% rename from lib/librdkafka-2.10.1/src/crc32c.c rename to lib/librdkafka-2.15.0/src/crc32c.c diff --git a/lib/librdkafka-2.10.1/src/crc32c.h b/lib/librdkafka-2.15.0/src/crc32c.h similarity index 100% rename from lib/librdkafka-2.10.1/src/crc32c.h rename to lib/librdkafka-2.15.0/src/crc32c.h diff --git a/lib/librdkafka-2.10.1/src/generate_proto.sh b/lib/librdkafka-2.15.0/src/generate_proto.sh similarity index 100% rename from lib/librdkafka-2.10.1/src/generate_proto.sh rename to lib/librdkafka-2.15.0/src/generate_proto.sh diff --git a/lib/librdkafka-2.10.1/src/librdkafka_cgrp_synch.png b/lib/librdkafka-2.15.0/src/librdkafka_cgrp_synch.png similarity index 100% rename from lib/librdkafka-2.10.1/src/librdkafka_cgrp_synch.png rename to lib/librdkafka-2.15.0/src/librdkafka_cgrp_synch.png diff --git a/lib/librdkafka-2.10.1/src/lz4.c b/lib/librdkafka-2.15.0/src/lz4.c similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4.c rename to lib/librdkafka-2.15.0/src/lz4.c diff --git a/lib/librdkafka-2.10.1/src/lz4.h b/lib/librdkafka-2.15.0/src/lz4.h similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4.h rename to lib/librdkafka-2.15.0/src/lz4.h diff --git a/lib/librdkafka-2.10.1/src/lz4frame.c b/lib/librdkafka-2.15.0/src/lz4frame.c similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4frame.c rename to lib/librdkafka-2.15.0/src/lz4frame.c diff --git a/lib/librdkafka-2.10.1/src/lz4frame.h b/lib/librdkafka-2.15.0/src/lz4frame.h similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4frame.h rename to lib/librdkafka-2.15.0/src/lz4frame.h diff --git a/lib/librdkafka-2.10.1/src/lz4frame_static.h b/lib/librdkafka-2.15.0/src/lz4frame_static.h similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4frame_static.h rename to lib/librdkafka-2.15.0/src/lz4frame_static.h diff --git a/lib/librdkafka-2.10.1/src/lz4hc.c b/lib/librdkafka-2.15.0/src/lz4hc.c similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4hc.c rename to lib/librdkafka-2.15.0/src/lz4hc.c diff --git a/lib/librdkafka-2.10.1/src/lz4hc.h b/lib/librdkafka-2.15.0/src/lz4hc.h similarity index 100% rename from lib/librdkafka-2.10.1/src/lz4hc.h rename to lib/librdkafka-2.15.0/src/lz4hc.h diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb.h b/lib/librdkafka-2.15.0/src/nanopb/pb.h similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb.h rename to lib/librdkafka-2.15.0/src/nanopb/pb.h diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb_common.c b/lib/librdkafka-2.15.0/src/nanopb/pb_common.c similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb_common.c rename to lib/librdkafka-2.15.0/src/nanopb/pb_common.c diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb_common.h b/lib/librdkafka-2.15.0/src/nanopb/pb_common.h similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb_common.h rename to lib/librdkafka-2.15.0/src/nanopb/pb_common.h diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb_decode.c b/lib/librdkafka-2.15.0/src/nanopb/pb_decode.c similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb_decode.c rename to lib/librdkafka-2.15.0/src/nanopb/pb_decode.c diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb_decode.h b/lib/librdkafka-2.15.0/src/nanopb/pb_decode.h similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb_decode.h rename to lib/librdkafka-2.15.0/src/nanopb/pb_decode.h diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb_encode.c b/lib/librdkafka-2.15.0/src/nanopb/pb_encode.c similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb_encode.c rename to lib/librdkafka-2.15.0/src/nanopb/pb_encode.c diff --git a/lib/librdkafka-2.10.1/src/nanopb/pb_encode.h b/lib/librdkafka-2.15.0/src/nanopb/pb_encode.h similarity index 100% rename from lib/librdkafka-2.10.1/src/nanopb/pb_encode.h rename to lib/librdkafka-2.15.0/src/nanopb/pb_encode.h diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/common.pb.c b/lib/librdkafka-2.15.0/src/opentelemetry/common.pb.c similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/common.pb.c rename to lib/librdkafka-2.15.0/src/opentelemetry/common.pb.c diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/common.pb.h b/lib/librdkafka-2.15.0/src/opentelemetry/common.pb.h similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/common.pb.h rename to lib/librdkafka-2.15.0/src/opentelemetry/common.pb.h diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/metrics.options b/lib/librdkafka-2.15.0/src/opentelemetry/metrics.options similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/metrics.options rename to lib/librdkafka-2.15.0/src/opentelemetry/metrics.options diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/metrics.pb.c b/lib/librdkafka-2.15.0/src/opentelemetry/metrics.pb.c similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/metrics.pb.c rename to lib/librdkafka-2.15.0/src/opentelemetry/metrics.pb.c diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/metrics.pb.h b/lib/librdkafka-2.15.0/src/opentelemetry/metrics.pb.h similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/metrics.pb.h rename to lib/librdkafka-2.15.0/src/opentelemetry/metrics.pb.h diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/resource.pb.c b/lib/librdkafka-2.15.0/src/opentelemetry/resource.pb.c similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/resource.pb.c rename to lib/librdkafka-2.15.0/src/opentelemetry/resource.pb.c diff --git a/lib/librdkafka-2.10.1/src/opentelemetry/resource.pb.h b/lib/librdkafka-2.15.0/src/opentelemetry/resource.pb.h similarity index 100% rename from lib/librdkafka-2.10.1/src/opentelemetry/resource.pb.h rename to lib/librdkafka-2.15.0/src/opentelemetry/resource.pb.h diff --git a/lib/librdkafka-2.10.1/src/queue.h b/lib/librdkafka-2.15.0/src/queue.h similarity index 100% rename from lib/librdkafka-2.10.1/src/queue.h rename to lib/librdkafka-2.15.0/src/queue.h diff --git a/lib/librdkafka-2.10.1/src/rd.h b/lib/librdkafka-2.15.0/src/rd.h similarity index 73% rename from lib/librdkafka-2.10.1/src/rd.h rename to lib/librdkafka-2.15.0/src/rd.h index a01d4d03c31..8c215fc6717 100644 --- a/lib/librdkafka-2.10.1/src/rd.h +++ b/lib/librdkafka-2.15.0/src/rd.h @@ -44,6 +44,11 @@ #ifndef _POSIX_C_SOURCE #define _POSIX_C_SOURCE 200809L /* for timespec on solaris */ #endif +#else +#ifndef _CRT_RAND_S +#define _CRT_RAND_S /* for rand_s() on MSVC. It needs to be defined before \ + * including . */ +#endif #endif #include @@ -53,6 +58,7 @@ #include #include #include +#include #include "tinycthread.h" #include "rdsysqueue.h" @@ -226,6 +232,12 @@ static RD_INLINE RD_UNUSED char *rd_strndup(const char *s, size_t len) { #define RD_ARRAYSIZE(A) RD_ARRAY_SIZE(A) #define RD_SIZEOF(TYPE, MEMBER) sizeof(((TYPE *)NULL)->MEMBER) #define RD_OFFSETOF(TYPE, MEMBER) ((size_t) & (((TYPE *)NULL)->MEMBER)) +/** Array foreach */ +#define RD_ARRAY_FOREACH_INDEX(ELEM, ARRAY, INDEX) \ + for ((INDEX = 0, (ELEM) = (ARRAY)[INDEX]); \ + INDEX < RD_ARRAY_SIZE(ARRAY); \ + (ELEM) = \ + (++INDEX < RD_ARRAY_SIZE(ARRAY) ? (ARRAY)[INDEX] : (ELEM))) /** * Returns the 'I'th array element from static sized array 'A' @@ -405,14 +417,17 @@ static RD_INLINE RD_UNUSED int rd_refcnt_get(rd_refcnt_t *R) { rd_refcnt_get(R), (R), WHAT, __FUNCTION__, __LINE__), \ rd_refcnt_sub0(R)) -#define rd_refcnt_sub(R) \ +#define rd_refcnt_sub_fl(FUNC, LINE, R) \ (fprintf(stderr, "REFCNT DEBUG: %-35s %d -1: %16p: %s:%d\n", #R, \ - rd_refcnt_get(R), (R), __FUNCTION__, __LINE__), \ + rd_refcnt_get(R), (R), (FUNC), (LINE)), \ rd_refcnt_sub0(R)) +#define rd_refcnt_sub(R) rd_refcnt_sub_fl(__FUNCTION__, __LINE__, R) + #else #define rd_refcnt_add_fl(FUNC, LINE, R) rd_refcnt_add0(R) #define rd_refcnt_add(R) rd_refcnt_add0(R) +#define rd_refcnt_sub_fl(FUNC, LINE, R) rd_refcnt_sub0(R) #define rd_refcnt_sub(R) rd_refcnt_sub0(R) #endif @@ -438,4 +453,140 @@ typedef struct rd_chariov_s { size_t size; } rd_chariov_t; +/** + * @brief Read the file at \p file_path in binary mode and return its contents. + * The returned buffer is NULL-terminated, + * the size parameter will contain the actual file size. + * + * @param file_path Path to the file to read. + * @param size Pointer to store the file size (optional). + * @param max_size Maximum file size to read (0 for no limit) (optional). + * + * @returns Newly allocated buffer containing the file contents. + * NULL on error (file not found, too large, etc). + * + * @remark The returned pointer ownership is transferred to the caller. + * + * @locality Any thread + */ +static RD_INLINE RD_UNUSED char * +rd_file_read(const char *file_path, size_t *size, size_t max_size) { + FILE *file; + char *buf = NULL; + size_t file_size; + size_t read_size; + if (!size) + size = &read_size; + +#ifndef _WIN32 + file = fopen(file_path, "rb"); +#else + file = NULL; + errno = fopen_s(&file, file_path, "rb"); +#endif + if (!file) + return NULL; + + if (fseek(file, 0, SEEK_END) != 0) + goto err; + + file_size = (size_t)ftell(file); + if (file_size < 0) + goto err; + + if (fseek(file, 0, SEEK_SET) != 0) + goto err; + + /* Check if file is too large */ + if (max_size > 0 && file_size > max_size) + goto err; + + /* Allocate buffer with extra byte for NULL terminator */ + buf = (char *)rd_malloc(file_size + 1); + read_size = fread(buf, 1, file_size, file); + + if (read_size != file_size) + goto err; + + /* NULL terminate the buffer */ + buf[file_size] = '\0'; + *size = file_size; + fclose(file); + return buf; +err: + fclose(file); + if (buf) + rd_free(buf); + return NULL; +} + +static RD_INLINE RD_UNUSED FILE * +rd_file_mkstemp(const char *prefix, + const char *mode, + char *tempfile_path_out, + size_t tempfile_path_out_size) { + FILE *tempfile; + +#ifdef _WIN32 + char tempfolder_path[MAX_PATH]; + char tempfile_path[MAX_PATH]; + if (!GetTempPathA(MAX_PATH, tempfolder_path)) + return NULL; /* Failed to get temp folder path */ + + + if (!GetTempFileNameA(tempfolder_path, "TMP", 1, tempfile_path)) + return NULL; /* Failed to create temp file name */ + + tempfile = fopen(tempfile_path, mode); +#else + int tempfile_fd; + char tempfile_path[512]; + rd_snprintf(tempfile_path, sizeof(tempfile_path), "/tmp/%sXXXXXX", + prefix); + tempfile_fd = mkstemp(tempfile_path); + if (tempfile_fd < 0) + return NULL; + + tempfile = fdopen(tempfile_fd, mode); +#endif + + if (!tempfile) + return NULL; + + if (tempfile_path_out) + rd_snprintf(tempfile_path_out, tempfile_path_out_size, "%s", + tempfile_path); + return tempfile; +} + +/** + * @brief Retrive stat for a \p path . + * + * @param path Path to the file or directory. + * @param is_dir Pointer to store if the \p path is a directory (optional). + * + * @return `rd_true` if the path exists. + */ +static RD_INLINE RD_UNUSED rd_bool_t rd_file_stat(const char *path, + rd_bool_t *is_dir) { +#ifdef _WIN32 + struct _stat st; + if (_stat(path, &st) == 0) { + if (is_dir) + *is_dir = st.st_mode & S_IFDIR; + return rd_true; + } +#else + struct stat st; + if (stat(path, &st) == 0) { + if (is_dir) + *is_dir = S_ISDIR(st.st_mode); + return rd_true; + } +#endif + if (is_dir) + *is_dir = rd_false; + return rd_false; +} + #endif /* _RD_H_ */ diff --git a/lib/librdkafka-2.10.1/src/rdaddr.c b/lib/librdkafka-2.15.0/src/rdaddr.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdaddr.c rename to lib/librdkafka-2.15.0/src/rdaddr.c diff --git a/lib/librdkafka-2.10.1/src/rdaddr.h b/lib/librdkafka-2.15.0/src/rdaddr.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdaddr.h rename to lib/librdkafka-2.15.0/src/rdaddr.h diff --git a/lib/librdkafka-2.10.1/src/rdatomic.h b/lib/librdkafka-2.15.0/src/rdatomic.h similarity index 74% rename from lib/librdkafka-2.10.1/src/rdatomic.h rename to lib/librdkafka-2.15.0/src/rdatomic.h index 4b97dd7d087..cad90f6c3c2 100644 --- a/lib/librdkafka-2.10.1/src/rdatomic.h +++ b/lib/librdkafka-2.15.0/src/rdatomic.h @@ -111,24 +111,25 @@ static RD_INLINE int32_t RD_UNUSED rd_atomic32_get(rd_atomic32_t *ra) { #endif } +/** + * @brief Set the atomic value to \p v and return the previous value to + * that can be used to determine if the value was changed. + */ static RD_INLINE int32_t RD_UNUSED rd_atomic32_set(rd_atomic32_t *ra, int32_t v) { #ifdef _WIN32 return InterlockedExchange((LONG *)&ra->val, v); -#elif !HAVE_ATOMICS_32 +#elif HAVE_ATOMICS_32 && HAVE_ATOMICS_32_ATOMIC + return __atomic_exchange_n(&ra->val, v, __ATOMIC_SEQ_CST); +#elif HAVE_ATOMICS_32 && HAVE_ATOMICS_32_SYNC + return __sync_lock_test_and_set(&ra->val, v); +#else int32_t r; mtx_lock(&ra->lock); - r = ra->val = v; + r = ra->val; + ra->val = v; mtx_unlock(&ra->lock); return r; -#elif HAVE_ATOMICS_32_ATOMIC - __atomic_store_n(&ra->val, v, __ATOMIC_SEQ_CST); - return v; -#elif HAVE_ATOMICS_32_SYNC - (void)__sync_lock_test_and_set(&ra->val, v); - return v; -#else - return ra->val = v; // FIXME #endif } @@ -200,26 +201,61 @@ static RD_INLINE int64_t RD_UNUSED rd_atomic64_get(rd_atomic64_t *ra) { #endif } - +/** + * @brief Set the atomic value to \p v and return the previous value to + * that can be used to determine if the value was changed. + */ static RD_INLINE int64_t RD_UNUSED rd_atomic64_set(rd_atomic64_t *ra, int64_t v) { #ifdef _WIN32 return InterlockedExchange64(&ra->val, v); -#elif !HAVE_ATOMICS_64 +#elif HAVE_ATOMICS_64 && HAVE_ATOMICS_64_ATOMIC + return __atomic_exchange_n(&ra->val, v, __ATOMIC_SEQ_CST); +#elif HAVE_ATOMICS_64 && HAVE_ATOMICS_64_SYNC + return __sync_lock_test_and_set(&ra->val, v); +#else int64_t r; mtx_lock(&ra->lock); - ra->val = v; r = ra->val; + ra->val = v; mtx_unlock(&ra->lock); return r; -#elif HAVE_ATOMICS_64_ATOMIC - __atomic_store_n(&ra->val, v, __ATOMIC_SEQ_CST); - return v; -#elif HAVE_ATOMICS_64_SYNC - (void)__sync_lock_test_and_set(&ra->val, v); - return v; +#endif +} + +/** + * @brief Atomic compare-and-swap on a 64-bit value. + * + * If *ra == @p expected, atomically replace it with @p desired + * and return 1. Otherwise leave *ra unchanged and return 0. + */ +static RD_INLINE int RD_UNUSED rd_atomic64_cas(rd_atomic64_t *ra, + int64_t expected, + int64_t desired) { +#ifdef __SUNPRO_C + return atomic_cas_64((volatile uint64_t *)&ra->val, (uint64_t)expected, + (uint64_t)desired) == (uint64_t)expected; +#elif defined(_WIN32) + return InterlockedCompareExchange64((LONG64 *)&ra->val, (LONG64)desired, + (LONG64)expected) == + (LONG64)expected; +#elif HAVE_ATOMICS_64 && HAVE_ATOMICS_64_ATOMIC + return __atomic_compare_exchange_n(&ra->val, &expected, desired, + 0 /* strong */, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); +#elif HAVE_ATOMICS_64 && HAVE_ATOMICS_64_SYNC + return __sync_bool_compare_and_swap(&ra->val, expected, desired); #else - return ra->val = v; // FIXME + int r; + mtx_lock(&ra->lock); + if (ra->val == expected) { + ra->val = desired; + r = 1; + } else { + r = 0; + } + mtx_unlock(&ra->lock); + return r; #endif } diff --git a/lib/librdkafka-2.10.1/src/rdavg.h b/lib/librdkafka-2.15.0/src/rdavg.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdavg.h rename to lib/librdkafka-2.15.0/src/rdavg.h diff --git a/lib/librdkafka-2.10.1/src/rdavl.c b/lib/librdkafka-2.15.0/src/rdavl.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdavl.c rename to lib/librdkafka-2.15.0/src/rdavl.c diff --git a/lib/librdkafka-2.10.1/src/rdavl.h b/lib/librdkafka-2.15.0/src/rdavl.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdavl.h rename to lib/librdkafka-2.15.0/src/rdavl.h diff --git a/lib/librdkafka-2.10.1/src/rdbase64.c b/lib/librdkafka-2.15.0/src/rdbase64.c similarity index 85% rename from lib/librdkafka-2.10.1/src/rdbase64.c rename to lib/librdkafka-2.15.0/src/rdbase64.c index 70c797136a1..7d87650903f 100644 --- a/lib/librdkafka-2.10.1/src/rdbase64.c +++ b/lib/librdkafka-2.15.0/src/rdbase64.c @@ -119,6 +119,37 @@ char *rd_base64_encode_str(const rd_chariov_t *in) { return out.ptr; } +/** + * @brief Base64 encode binary input \p in and return a newly allocated, + * base64-encoded string with URL-safe characters. + * @returns a newly allocated, base64-encoded string or NULL in case of some + * issue with the conversion or the conversion is not supported. + * + * @remark Returned string must be freed after use. + */ +char *rd_base64_encode_str_urlsafe(const rd_chariov_t *in) { + rd_chariov_t out; + char *p; + rd_base64_encode(in, &out); + + /* Replace + with - and / with _ */ + for (p = out.ptr; *p; p++) { + if (*p == '+') + *p = '-'; + else if (*p == '/') + *p = '_'; + } + + /* Remove padding '=' characters */ + int newlen = strlen(out.ptr); + while (newlen > 0 && out.ptr[newlen - 1] == '=') { + out.ptr[newlen - 1] = '\0'; + newlen--; + } + + out.size = newlen; + return out.ptr; +} /** * @brief Base64 decode input string \p in. Ignores leading and trailing @@ -166,4 +197,4 @@ int rd_base64_decode(const rd_chariov_t *in, rd_chariov_t *out) { #else return -2; #endif -} \ No newline at end of file +} diff --git a/lib/librdkafka-2.10.1/src/rdbase64.h b/lib/librdkafka-2.15.0/src/rdbase64.h similarity index 96% rename from lib/librdkafka-2.10.1/src/rdbase64.h rename to lib/librdkafka-2.15.0/src/rdbase64.h index fd9e7a209fe..1fb12885488 100644 --- a/lib/librdkafka-2.10.1/src/rdbase64.h +++ b/lib/librdkafka-2.15.0/src/rdbase64.h @@ -36,6 +36,8 @@ void rd_base64_encode(const rd_chariov_t *in, rd_chariov_t *out); char *rd_base64_encode_str(const rd_chariov_t *in); +char *rd_base64_encode_str_urlsafe(const rd_chariov_t *in); + int rd_base64_decode(const rd_chariov_t *in, rd_chariov_t *out); #endif /* _RDBASE64_H_ */ \ No newline at end of file diff --git a/lib/librdkafka-2.10.1/src/rdbuf.c b/lib/librdkafka-2.15.0/src/rdbuf.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdbuf.c rename to lib/librdkafka-2.15.0/src/rdbuf.c diff --git a/lib/librdkafka-2.10.1/src/rdbuf.h b/lib/librdkafka-2.15.0/src/rdbuf.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdbuf.h rename to lib/librdkafka-2.15.0/src/rdbuf.h diff --git a/lib/librdkafka-2.10.1/src/rdcrc32.c b/lib/librdkafka-2.15.0/src/rdcrc32.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdcrc32.c rename to lib/librdkafka-2.15.0/src/rdcrc32.c diff --git a/lib/librdkafka-2.10.1/src/rdcrc32.h b/lib/librdkafka-2.15.0/src/rdcrc32.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdcrc32.h rename to lib/librdkafka-2.15.0/src/rdcrc32.h diff --git a/lib/librdkafka-2.10.1/src/rddl.c b/lib/librdkafka-2.15.0/src/rddl.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rddl.c rename to lib/librdkafka-2.15.0/src/rddl.c diff --git a/lib/librdkafka-2.10.1/src/rddl.h b/lib/librdkafka-2.15.0/src/rddl.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rddl.h rename to lib/librdkafka-2.15.0/src/rddl.h diff --git a/lib/librdkafka-2.10.1/src/rdendian.h b/lib/librdkafka-2.15.0/src/rdendian.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdendian.h rename to lib/librdkafka-2.15.0/src/rdendian.h diff --git a/lib/librdkafka-2.10.1/src/rdfloat.h b/lib/librdkafka-2.15.0/src/rdfloat.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdfloat.h rename to lib/librdkafka-2.15.0/src/rdfloat.h diff --git a/lib/librdkafka-2.10.1/src/rdfnv1a.c b/lib/librdkafka-2.15.0/src/rdfnv1a.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdfnv1a.c rename to lib/librdkafka-2.15.0/src/rdfnv1a.c diff --git a/lib/librdkafka-2.10.1/src/rdfnv1a.h b/lib/librdkafka-2.15.0/src/rdfnv1a.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdfnv1a.h rename to lib/librdkafka-2.15.0/src/rdfnv1a.h diff --git a/lib/librdkafka-2.10.1/src/rdgz.c b/lib/librdkafka-2.15.0/src/rdgz.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdgz.c rename to lib/librdkafka-2.15.0/src/rdgz.c diff --git a/lib/librdkafka-2.10.1/src/rdgz.h b/lib/librdkafka-2.15.0/src/rdgz.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdgz.h rename to lib/librdkafka-2.15.0/src/rdgz.h diff --git a/lib/librdkafka-2.10.1/src/rdhdrhistogram.c b/lib/librdkafka-2.15.0/src/rdhdrhistogram.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdhdrhistogram.c rename to lib/librdkafka-2.15.0/src/rdhdrhistogram.c diff --git a/lib/librdkafka-2.10.1/src/rdhdrhistogram.h b/lib/librdkafka-2.15.0/src/rdhdrhistogram.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdhdrhistogram.h rename to lib/librdkafka-2.15.0/src/rdhdrhistogram.h diff --git a/lib/librdkafka-2.15.0/src/rdhttp.c b/lib/librdkafka-2.15.0/src/rdhttp.c new file mode 100644 index 00000000000..71738b9db3c --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdhttp.c @@ -0,0 +1,831 @@ +/* + * librdkafka - The Apache Kafka C/C++ library + * + * Copyright (c) 2021-2022, Magnus Edenhill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +/** + * @name HTTP client + * + */ + +#include "rdkafka_int.h" +#include "rdunittest.h" + +#include + +#include +#include "rdhttp.h" + +#if WITH_SSL +#include "rdkafka_ssl.h" +#endif + +/** Maximum response size, increase as necessary. */ +#define RD_HTTP_RESPONSE_SIZE_MAX 1024 * 1024 * 500 /* 500kb */ + + +void rd_http_error_destroy(rd_http_error_t *herr) { + rd_free(herr); +} + +static rd_http_error_t *rd_http_error_new(int code, const char *fmt, ...) + RD_FORMAT(printf, 2, 3); +static rd_http_error_t *rd_http_error_new(int code, const char *fmt, ...) { + size_t len = 0; + rd_http_error_t *herr; + va_list ap; + + va_start(ap, fmt); + + if (fmt && *fmt) { + va_list ap2; + va_copy(ap2, ap); + len = rd_vsnprintf(NULL, 0, fmt, ap2); + va_end(ap2); + } + + /* Use single allocation for both herr and the error string */ + herr = rd_malloc(sizeof(*herr) + len + 1); + herr->code = code; + herr->errstr = herr->data; + + if (len > 0) + rd_vsnprintf(herr->errstr, len + 1, fmt, ap); + else + herr->errstr[0] = '\0'; + + va_end(ap); + + return herr; +} + +/** + * @brief Same as rd_http_error_new() but reads the error string from the + * provided buffer. + */ +static rd_http_error_t *rd_http_error_new_from_buf(int code, + const rd_buf_t *rbuf) { + rd_http_error_t *herr; + rd_slice_t slice; + size_t len = rd_buf_len(rbuf); + + if (len == 0) + return rd_http_error_new( + code, "Server did not provide an error string"); + + + /* Use single allocation for both herr and the error string */ + herr = rd_malloc(sizeof(*herr) + len + 1); + herr->code = code; + herr->errstr = herr->data; + rd_slice_init_full(&slice, rbuf); + rd_slice_read(&slice, herr->errstr, len); + herr->errstr[len] = '\0'; + + return herr; +} + +void rd_http_req_destroy(rd_http_req_t *hreq) { + RD_IF_FREE(hreq->hreq_curl, curl_easy_cleanup); + RD_IF_FREE(hreq->hreq_buf, rd_buf_destroy_free); +} + + +/** + * @brief Curl writefunction. Writes the bytes passed from curl + * to the hreq's buffer. + */ +static size_t +rd_http_req_write_cb(char *ptr, size_t size, size_t nmemb, void *userdata) { + rd_http_req_t *hreq = (rd_http_req_t *)userdata; + + if (unlikely(rd_buf_len(hreq->hreq_buf) + nmemb > + RD_HTTP_RESPONSE_SIZE_MAX)) + return 0; /* FIXME: Set some overflow flag or rely on curl? */ + + rd_buf_write(hreq->hreq_buf, ptr, nmemb); + + return nmemb; +} + +#if WITH_SSL +/** + * @brief Callback function for setting up the SSL_CTX for HTTPS requests. + * + * This function sets the default CA paths for the SSL_CTX, and if that fails, + * it attempts to probe and set a default CA location. If `probe` is forced + * it skips the default CA paths and directly probes for CA certificates. + * + * On Windows, it attempts to load CA root certificates from the + * configured Windows certificate stores before falling back to the default. + * + * @return `CURLE_OK` on success, or `CURLE_SSL_CACERT_BADFILE` on failure. + */ +static CURLcode +rd_http_ssl_ctx_function(CURL *curl, void *sslctx, void *userptr) { + SSL_CTX *ctx = (SSL_CTX *)sslctx; + rd_kafka_t *rk = (rd_kafka_t *)userptr; + int r = -1; + rd_bool_t force_probe = + !rd_strcmp(rk->rk_conf.https.ca_location, "probe"); + rd_bool_t use_probe = force_probe; + +#if WITH_STATIC_LIB_libcrypto + /* We fallback to `probe` when statically linked. */ + use_probe = rd_true; +#endif + +#ifdef _WIN32 + /* Attempt to load CA root certificates from the + * configured Windows certificate stores. */ + r = rd_kafka_ssl_win_load_cert_stores(rk, "https", ctx, + rk->rk_conf.ssl.ca_cert_stores); + if (r == 0) { + rd_kafka_log(rk, LOG_NOTICE, "CERTSTORE", + "No CA certificates loaded for `https` from " + "Windows certificate stores: " + "falling back to default OpenSSL CA paths"); + r = -1; + } else if (r == -1) + rd_kafka_log(rk, LOG_NOTICE, "CERTSTORE", + "Failed to load CA certificates for `https` from " + "Windows certificate stores: " + "falling back to default OpenSSL CA paths"); + + if (r != -1) { + rd_kafka_dbg(rk, SECURITY, "SSL", + "Successfully loaded CA certificates for `https` " + "from Windows certificate stores"); + return CURLE_OK; /* Success, CA certs loaded on Windows */ + } +#endif + + if (!force_probe) { + /* Previous default behavior: use predefined paths set when + * building OpenSSL. */ + char errstr[512]; + r = SSL_CTX_set_default_verify_paths(ctx); + if (r == 1) { + rd_kafka_dbg(rk, SECURITY, "SSL", + "SSL_CTX_set_default_verify_paths() " + "for `https` " + "succeeded"); + return CURLE_OK; /* Success */ + } + + /* Read error and clear the error stack. */ + rd_kafka_ssl_error0(rk, NULL, "https", errstr, sizeof(errstr)); + rd_kafka_dbg(rk, SECURITY, "SSL", + "SSL_CTX_set_default_verify_paths() " + "for `https` " + "failed: %s", + errstr); + } + + if (use_probe) { + /* We asked for probing or we're using + * a statically linked version of OpenSSL. */ + + r = rd_kafka_ssl_probe_and_set_default_ca_location(rk, "https", + ctx); + if (r == 0) + return CURLE_OK; + } + + return CURLE_SSL_CACERT_BADFILE; +} + +static void rd_http_ssl_configure(rd_kafka_t *rk, CURL *hreq_curl) { + rd_bool_t force_probe = + !rd_strcmp(rk->rk_conf.https.ca_location, "probe"); + + if (!force_probe && rk->rk_conf.https.ca_location) { + rd_bool_t is_dir; + rd_kafka_dbg(rk, SECURITY, "SSL", + "Setting `https` CA certs from " + "configured location: %s", + rk->rk_conf.https.ca_location); + if (rd_file_stat(rk->rk_conf.https.ca_location, &is_dir)) { + if (is_dir) { + curl_easy_setopt(hreq_curl, CURLOPT_CAPATH, + rk->rk_conf.https.ca_location); + curl_easy_setopt(hreq_curl, CURLOPT_CAINFO, + NULL); + } else { + curl_easy_setopt(hreq_curl, CURLOPT_CAPATH, + NULL); + curl_easy_setopt(hreq_curl, CURLOPT_CAINFO, + rk->rk_conf.https.ca_location); + } + } else { + /* Path doesn't exist, don't set any trusted + * certificate. */ + curl_easy_setopt(hreq_curl, CURLOPT_CAINFO, NULL); + curl_easy_setopt(hreq_curl, CURLOPT_CAPATH, NULL); + } + } else if (!force_probe && rk->rk_conf.https.ca_pem) { +#if CURL_AT_LEAST_VERSION(7, 77, 0) + struct curl_blob ca_blob = { + .data = rk->rk_conf.https.ca_pem, + .len = strlen(rk->rk_conf.https.ca_pem), + .flags = CURL_BLOB_COPY}; + rd_kafka_dbg(rk, SECURITY, "SSL", + "Setting `https` CA certs from " + "configured PEM string"); + curl_easy_setopt(hreq_curl, CURLOPT_CAINFO_BLOB, &ca_blob); +#endif + /* Only the blob should be set, no default paths. */ + curl_easy_setopt(hreq_curl, CURLOPT_CAINFO, NULL); + curl_easy_setopt(hreq_curl, CURLOPT_CAPATH, NULL); + } else { + curl_easy_setopt(hreq_curl, CURLOPT_SSL_CTX_FUNCTION, + rd_http_ssl_ctx_function); + curl_easy_setopt(hreq_curl, CURLOPT_SSL_CTX_DATA, rk); + } +} +#endif + +/** + * @brief Check if the error returned from HTTP(S) is temporary or not. + * + * @returns If the \p error_code is temporary, return rd_true, + * otherwise return rd_false. + * + * @locality Any thread. + * @locks None. + * @locks_acquired None. + */ +static rd_bool_t rd_http_is_failure_temporary(int error_code) { + switch (error_code) { + case 408: /**< Request timeout */ + case 425: /**< Too early */ + case 429: /**< Too many requests */ + case 500: /**< Internal server error */ + case 502: /**< Bad gateway */ + case 503: /**< Service unavailable */ + case 504: /**< Gateway timeout */ + return rd_true; + + default: + return rd_false; + } +} + +rd_http_error_t * +rd_http_req_init(rd_kafka_t *rk, rd_http_req_t *hreq, const char *url) { + memset(hreq, 0, sizeof(*hreq)); + + hreq->hreq_curl = curl_easy_init(); + if (!hreq->hreq_curl) + return rd_http_error_new(-1, "Failed to create curl handle"); + + hreq->hreq_buf = rd_buf_new(1, 1024); + + curl_easy_setopt(hreq->hreq_curl, CURLOPT_URL, url); +#if CURL_AT_LEAST_VERSION(7, 85, 0) + curl_easy_setopt(hreq->hreq_curl, CURLOPT_PROTOCOLS_STR, "http,https"); +#else + /* As of 06/10/2025 Debian 10 and CentOS Stream 9 ship with + * older CURL versions, remove this condition once they're not supported + * anymore. */ + curl_easy_setopt(hreq->hreq_curl, CURLOPT_PROTOCOLS, + CURLPROTO_HTTP | CURLPROTO_HTTPS); +#endif + curl_easy_setopt(hreq->hreq_curl, CURLOPT_MAXREDIRS, 16L); + curl_easy_setopt(hreq->hreq_curl, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(hreq->hreq_curl, CURLOPT_ERRORBUFFER, + hreq->hreq_curl_errstr); + curl_easy_setopt(hreq->hreq_curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(hreq->hreq_curl, CURLOPT_WRITEFUNCTION, + rd_http_req_write_cb); + curl_easy_setopt(hreq->hreq_curl, CURLOPT_WRITEDATA, (void *)hreq); + +#if WITH_SSL + rd_http_ssl_configure(rk, hreq->hreq_curl); +#endif + + return NULL; +} + +/** + * @brief Synchronously (blockingly) perform the HTTP operation. + */ +rd_http_error_t *rd_http_req_perform_sync(rd_http_req_t *hreq) { + CURLcode res; + long code = 0; + + res = curl_easy_perform(hreq->hreq_curl); + if (unlikely(res != CURLE_OK)) + return rd_http_error_new(-1, "%s", hreq->hreq_curl_errstr); + + curl_easy_getinfo(hreq->hreq_curl, CURLINFO_RESPONSE_CODE, &code); + hreq->hreq_code = (int)code; + if (hreq->hreq_code >= 400) + return rd_http_error_new_from_buf(hreq->hreq_code, + hreq->hreq_buf); + + return NULL; +} + + +int rd_http_req_get_code(const rd_http_req_t *hreq) { + return hreq->hreq_code; +} + +const char *rd_http_req_get_content_type(rd_http_req_t *hreq) { + const char *content_type = NULL; + + if (curl_easy_getinfo(hreq->hreq_curl, CURLINFO_CONTENT_TYPE, + &content_type)) + return NULL; + + return content_type; +} + + +/** + * @brief Perform a blocking HTTP(S) request to \p url. + * Retries the request \p retries times with linear backoff. + * Interval of \p retry_ms milliseconds is used between retries. + * + * @param url The URL to perform the request to. + * @param headers_array Array of HTTP(S) headers to set, each element + * is a string in the form "key: value" + * @param headers_array_cnt Number of elements in \p headers_array. + * @param timeout_s Timeout in seconds for the request, 0 means default + * `rd_http_req_init()` timeout. + * @param retries Number of retries to perform on failure. + * @param retry_ms Milliseconds to wait between retries. + * @param rbufp (out) Pointer to a buffer that will be filled with the response. + * @param content_type (out, optional) Pointer to a string that will be filled + * with the content type of the response, if not NULL. + * @param response_code (out, optional) Pointer to an integer that will be + * filled with the HTTP response code, if not NULL. + * + * @return Returns NULL on success (HTTP response code < 400), or an error + * object on transport or HTTP error. + * + * @remark Returned error object, when non-NULL, must be destroyed + * by calling rd_http_error_destroy(). + * + * @locality Any thread. + * @locks None. + * @locks_acquired None. + */ +rd_http_error_t *rd_http_get(rd_kafka_t *rk, + const char *url, + char **headers_array, + size_t headers_array_cnt, + int timeout_s, + int retries, + int retry_ms, + rd_buf_t **rbufp, + char **content_type, + int *response_code) { + rd_http_req_t hreq; + rd_http_error_t *herr = NULL; + struct curl_slist *headers = NULL; + char *header; + int i; + size_t len, j; + + *rbufp = NULL; + if (content_type) + *content_type = NULL; + if (response_code) + *response_code = -1; + + herr = rd_http_req_init(rk, &hreq, url); + if (unlikely(herr != NULL)) + return herr; + + for (j = 0; j < headers_array_cnt; j++) { + header = headers_array[j]; + if (header && *header) + headers = curl_slist_append(headers, header); + } + curl_easy_setopt(hreq.hreq_curl, CURLOPT_HTTPHEADER, headers); + if (timeout_s > 0) + curl_easy_setopt(hreq.hreq_curl, CURLOPT_TIMEOUT, + (long)timeout_s); + + for (i = 0; i <= retries; i++) { + if (rd_kafka_terminating(rk)) { + herr = rd_http_error_new(-1, "Terminating"); + goto done; + } + + herr = rd_http_req_perform_sync(&hreq); + len = rd_buf_len(hreq.hreq_buf); + + if (!herr) { + if (len > 0) + break; /* Success */ + /* Empty response */ + goto done; + } + + /* Retry if HTTP(S) request returns temporary error and there + * are remaining retries, else fail. */ + if (i == retries || !rd_http_is_failure_temporary(herr->code)) { + goto done; + } + + /* Retry */ + rd_http_error_destroy(herr); + rd_usleep(retry_ms * 1000 * (i + 1), &rk->rk_terminate); + } + + *rbufp = hreq.hreq_buf; + hreq.hreq_buf = NULL; + + if (content_type) { + const char *ct = rd_http_req_get_content_type(&hreq); + if (ct && *ct) + *content_type = rd_strdup(ct); + else + *content_type = NULL; + } + if (response_code) + *response_code = hreq.hreq_code; + +done: + RD_IF_FREE(headers, curl_slist_free_all); + rd_http_req_destroy(&hreq); + return herr; +} + + +/** + * @brief Extract the JSON object from \p hreq and return it in \p *jsonp. + * + * @returns Returns NULL on success, or an JSON parsing error - this + * error object must be destroyed by calling rd_http_error_destroy(). + */ +rd_http_error_t *rd_http_parse_json(rd_http_req_t *hreq, cJSON **jsonp) { + size_t len; + char *raw_json; + const char *end = NULL; + rd_slice_t slice; + rd_http_error_t *herr = NULL; + + /* cJSON requires the entire input to parse in contiguous memory. */ + rd_slice_init_full(&slice, hreq->hreq_buf); + len = rd_buf_len(hreq->hreq_buf); + + raw_json = rd_malloc(len + 1); + rd_slice_read(&slice, raw_json, len); + raw_json[len] = '\0'; + + /* Parse JSON */ + *jsonp = cJSON_ParseWithOpts(raw_json, &end, 0); + + if (!*jsonp) + herr = rd_http_error_new(hreq->hreq_code, + "Failed to parse JSON response " + "at %" PRIusz "/%" PRIusz, + (size_t)(end - raw_json), len); + rd_free(raw_json); + return herr; +} + + +/** + * @brief Perform a blocking HTTP(S) request to \p url with + * HTTP(S) headers and data with \p timeout_s. + * If the HTTP(S) request fails, will retry another \p retries times + * with multiplying backoff \p retry_ms. + * + * @returns The result will be returned in \p *jsonp. + * Returns NULL on success (HTTP response code < 400), or an error + * object on transport, HTTP error or a JSON parsing error - this + * error object must be destroyed by calling rd_http_error_destroy(). + * + * @locality Any thread. + */ +rd_http_error_t *rd_http_post_expect_json(rd_kafka_t *rk, + const char *url, + const struct curl_slist *headers, + const char *post_fields, + size_t post_fields_size, + int timeout_s, + int retries, + int retry_ms, + cJSON **jsonp) { + rd_http_error_t *herr; + rd_http_req_t hreq; + int i; + size_t len; + const char *content_type; + + herr = rd_http_req_init(rk, &hreq, url); + if (unlikely(herr != NULL)) + return herr; + + curl_easy_setopt(hreq.hreq_curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hreq.hreq_curl, CURLOPT_TIMEOUT, (long)timeout_s); + + curl_easy_setopt(hreq.hreq_curl, CURLOPT_POSTFIELDSIZE, + post_fields_size); + curl_easy_setopt(hreq.hreq_curl, CURLOPT_POSTFIELDS, post_fields); + + for (i = 0; i <= retries; i++) { + if (rd_kafka_terminating(rk)) { + rd_http_req_destroy(&hreq); + return rd_http_error_new(-1, "Terminating"); + } + + herr = rd_http_req_perform_sync(&hreq); + len = rd_buf_len(hreq.hreq_buf); + + if (!herr) { + if (len > 0) + break; /* Success */ + /* Empty response */ + rd_http_req_destroy(&hreq); + return NULL; + } + /* Retry if HTTP(S) request returns temporary error and there + * are remaining retries, else fail. */ + if (i == retries || !rd_http_is_failure_temporary(herr->code)) { + rd_http_req_destroy(&hreq); + return herr; + } + + /* Retry */ + rd_http_error_destroy(herr); + rd_usleep(retry_ms * 1000 * (i + 1), &rk->rk_terminate); + } + + content_type = rd_http_req_get_content_type(&hreq); + + if (!content_type || rd_strncasecmp(content_type, "application/json", + strlen("application/json"))) { + if (!herr) + herr = rd_http_error_new( + hreq.hreq_code, "Response is not JSON encoded: %s", + content_type ? content_type : "(n/a)"); + rd_http_req_destroy(&hreq); + return herr; + } + + herr = rd_http_parse_json(&hreq, jsonp); + + rd_http_req_destroy(&hreq); + + return herr; +} + +/** + * @brief Append \p params to \p url, taking care of existing query parameters + * and hash fragments. \p params must be already URL encoded. + * + * @returns A newly allocated string with the appended parameters or NULL + * on error. + */ +char *rd_http_get_params_append(const char *url, const char *params) { + char *new_url; + CURLU *u = curl_url(); + CURLUcode rc = curl_url_set(u, CURLUPART_URL, url, 0); + if (rc != CURLUE_OK) + goto err; + + rc = curl_url_set(u, CURLUPART_QUERY, params, CURLU_APPENDQUERY); + if (rc != CURLUE_OK) + goto err; + + rc = curl_url_set(u, CURLUPART_FRAGMENT, NULL, + 0); // remove hash fragment + if (rc != CURLUE_OK) + goto err; + + rc = curl_url_get(u, CURLUPART_URL, &new_url, 0); + if (rc != CURLUE_OK) + goto err; + + curl_url_cleanup(u); + return new_url; +err: + curl_url_cleanup(u); + return NULL; +} + +/** + * @brief Same as rd_http_get() but requires a JSON response. + * The response is parsed and a JSON object is returned in \p *jsonp. + * + * Same error semantics as rd_http_get(). + */ +rd_http_error_t *rd_http_get_json(rd_kafka_t *rk, + const char *url, + char **headers_array, + size_t headers_array_cnt, + int timeout_s, + int retries, + int retry_ms, + cJSON **jsonp) { + rd_http_error_t *herr; + int response_code; + char *content_type; + rd_buf_t *rbuf; + char **headers_array_new = + rd_calloc(headers_array_cnt + 1, sizeof(*headers_array_new)); + rd_http_req_t hreq; + + *jsonp = NULL; + + memcpy(headers_array_new, headers_array, + headers_array_cnt * sizeof(*headers_array_new)); + headers_array_new[headers_array_cnt++] = "Accept: application/json"; + + herr = rd_http_get(rk, url, headers_array_new, headers_array_cnt, + timeout_s, retries, retry_ms, &rbuf, &content_type, + &response_code); + rd_free(headers_array_new); + + if (unlikely(herr != NULL)) + return herr; + + if (!content_type || rd_strncasecmp(content_type, "application/json", + strlen("application/json"))) { + herr = rd_http_error_new(response_code, + "Response is not JSON encoded: %s", + content_type ? content_type : "(n/a)"); + RD_IF_FREE(rbuf, rd_buf_destroy_free); + RD_IF_FREE(content_type, rd_free); + return herr; + } + + hreq.hreq_buf = rbuf; + hreq.hreq_code = response_code; + herr = rd_http_parse_json(&hreq, jsonp); + RD_IF_FREE(rbuf, rd_buf_destroy_free); + RD_IF_FREE(content_type, rd_free); + + return herr; +} + + +void rd_http_global_init(void) { + curl_global_init(CURL_GLOBAL_DEFAULT); +} + + +int unittest_http_get(void) { + const char *base_url = rd_getenv("RD_UT_HTTP_URL", NULL); + char *error_url; + size_t error_url_size; + cJSON *json, *jval; + rd_http_error_t *herr; + rd_bool_t empty; + rd_kafka_t *rk; + + if (!base_url || !*base_url) + RD_UT_SKIP("RD_UT_HTTP_URL environment variable not set"); + + RD_UT_BEGIN(); + + rk = rd_calloc(1, sizeof(*rk)); + error_url_size = strlen(base_url) + strlen("/error") + 1; + error_url = rd_alloca(error_url_size); + rd_snprintf(error_url, error_url_size, "%s/error", base_url); + + /* Try the base url first, parse its JSON and extract a key-value. */ + json = NULL; + herr = rd_http_get_json(rk, base_url, NULL, 0, 5, 1, 1000, &json); + RD_UT_ASSERT(!herr, "Expected get_json(%s) to succeed, got: %s", + base_url, herr->errstr); + + empty = rd_true; + cJSON_ArrayForEach(jval, json) { + empty = rd_false; + break; + } + RD_UT_ASSERT(!empty, "Expected non-empty JSON response from %s", + base_url); + RD_UT_SAY( + "URL %s returned no error and a non-empty " + "JSON object/array as expected", + base_url); + cJSON_Delete(json); + + + /* Try the error URL, verify error code. */ + json = NULL; + herr = rd_http_get_json(rk, error_url, NULL, 0, 5, 1, 1000, &json); + RD_UT_ASSERT(herr != NULL, "Expected get_json(%s) to fail", error_url); + RD_UT_ASSERT(herr->code >= 400, + "Expected get_json(%s) error code >= " + "400, got %d", + error_url, herr->code); + RD_UT_SAY( + "Error URL %s returned code %d, errstr \"%s\" " + "and %s JSON object as expected", + error_url, herr->code, herr->errstr, json ? "a" : "no"); + /* Check if there's a JSON document returned */ + if (json) + cJSON_Delete(json); + rd_http_error_destroy(herr); + rd_free(rk); + + RD_UT_PASS(); +} + +int unittest_http_get_params_append(void) { + rd_kafka_t *rk; + char *res; + RD_UT_BEGIN(); + char *tests[] = {"http://localhost:1234", + "", + "http://localhost:1234/", + + "http://localhost:1234/", + "a=1", + "http://localhost:1234/?a=1", + + "https://localhost:1234/", + "a=1&b=2", + "https://localhost:1234/?a=1&b=2", + + "http://mydomain.com/?a=1", + "c=hi", + "http://mydomain.com/?a=1&c=hi", + + "https://mydomain.com/?", + "c=hi", + "https://mydomain.com/?c=hi", + + "http://localhost:1234/path?a=1&b=2#&c=3", + "c=hi", + "http://localhost:1234/path?a=1&b=2&c=hi", + + "http://localhost:1234#?c=3", + "a=1", + "http://localhost:1234/?a=1", + + "https://otherdomain.io/path?a=1&#c=3", + "b=2", + "https://otherdomain.io/path?a=1&b=2", + NULL}; + + + res = rd_http_get_params_append("", ""); + RD_UT_ASSERT(!res, "Expected NULL result, got: \"%s\"", res); + res = rd_http_get_params_append("", "a=2&b=3"); + RD_UT_ASSERT(!res, "Expected NULL result, got: \"%s\"", res); + + char **test = tests; + rk = rd_calloc(1, sizeof(*rk)); + while (test[0]) { + res = rd_http_get_params_append(test[0], test[1]); + RD_UT_ASSERT(!strcmp(res, test[2]), + "Expected \"%s\", got: \"%s\"", test[2], res); + rd_free(res); + test += 3; + } + rd_free(rk); + + RD_UT_PASS(); +} + +/** + * @brief Unittest. Requires a (local) webserver to be set with env var + * RD_UT_HTTP_URL=http://localhost:1234/some-path + * + * This server must return a JSON object or array containing at least one + * object on the main URL with a 2xx response code, + * and 4xx response on $RD_UT_HTTP_URL/error (with whatever type of body). + */ + +int unittest_http(void) { + int fails = 0; + + fails += unittest_http_get(); + fails += unittest_http_get_params_append(); + + return fails; +} diff --git a/lib/librdkafka-2.10.1/src/rdhttp.h b/lib/librdkafka-2.15.0/src/rdhttp.h similarity index 73% rename from lib/librdkafka-2.10.1/src/rdhttp.h rename to lib/librdkafka-2.15.0/src/rdhttp.h index 9691cc800e4..eb93006bf8b 100644 --- a/lib/librdkafka-2.10.1/src/rdhttp.h +++ b/lib/librdkafka-2.15.0/src/rdhttp.h @@ -33,7 +33,6 @@ #define CJSON_HIDE_SYMBOLS #include "cJSON.h" - typedef struct rd_http_error_s { int code; char *errstr; @@ -42,8 +41,26 @@ typedef struct rd_http_error_s { void rd_http_error_destroy(rd_http_error_t *herr); -rd_http_error_t *rd_http_get(const char *url, rd_buf_t **rbufp); -rd_http_error_t *rd_http_get_json(const char *url, cJSON **jsonp); +char *rd_http_get_params_append(const char *url, const char *params); + +rd_http_error_t *rd_http_get(rd_kafka_t *rk, + const char *url, + char **headers_array, + size_t headers_array_cnt, + int timeout_s, + int retries, + int retry_ms, + rd_buf_t **rbufp, + char **content_type, + int *response_code); +rd_http_error_t *rd_http_get_json(rd_kafka_t *rk, + const char *url, + char **headers_array, + size_t headers_array_cnt, + int timeout_s, + int retries, + int retry_ms, + cJSON **jsonp); void rd_http_global_init(void); @@ -62,7 +79,8 @@ typedef struct rd_http_req_s { * write to. */ } rd_http_req_t; -rd_http_error_t *rd_http_req_init(rd_http_req_t *hreq, const char *url); +rd_http_error_t * +rd_http_req_init(rd_kafka_t *rk, rd_http_req_t *hreq, const char *url); rd_http_error_t *rd_http_req_perform_sync(rd_http_req_t *hreq); rd_http_error_t *rd_http_parse_json(rd_http_req_t *hreq, cJSON **jsonp); rd_http_error_t *rd_http_post_expect_json(rd_kafka_t *rk, @@ -71,7 +89,7 @@ rd_http_error_t *rd_http_post_expect_json(rd_kafka_t *rk, const char *data_to_token, size_t data_to_token_size, int timeout_s, - int retry, + int retries, int retry_ms, cJSON **jsonp); void rd_http_req_destroy(rd_http_req_t *hreq); diff --git a/lib/librdkafka-2.10.1/src/rdinterval.h b/lib/librdkafka-2.15.0/src/rdinterval.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdinterval.h rename to lib/librdkafka-2.15.0/src/rdinterval.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka.c b/lib/librdkafka-2.15.0/src/rdkafka.c similarity index 69% rename from lib/librdkafka-2.10.1/src/rdkafka.c rename to lib/librdkafka-2.15.0/src/rdkafka.c index 03a1399cf61..a8691f0a3a7 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka.c +++ b/lib/librdkafka-2.15.0/src/rdkafka.c @@ -57,6 +57,7 @@ #include "rdkafka_interceptor.h" #include "rdkafka_idempotence.h" #include "rdkafka_sasl_oauthbearer.h" +#include "rdkafka_share_acknowledgement.h" #if WITH_OAUTHBEARER_OIDC #include "rdkafka_sasl_oauthbearer_oidc.h" #endif @@ -82,8 +83,14 @@ #endif -static once_flag rd_kafka_global_init_once = ONCE_FLAG_INIT; -static once_flag rd_kafka_global_srand_once = ONCE_FLAG_INIT; +static once_flag rd_kafka_global_init_once = ONCE_FLAG_INIT; +#ifdef _WIN32 +/* On Windows srand needs to be called on each thread. */ +static RD_TLS once_flag rd_kafka_srand_once = ONCE_FLAG_INIT; +#else +static once_flag rd_kafka_srand_once = ONCE_FLAG_INIT; +#endif + /** * @brief Global counter+lock for all active librdkafka instances @@ -130,6 +137,13 @@ void rd_kafka_set_thread_name(const char *fmt, ...) { */ static char RD_TLS rd_kafka_thread_sysname[16] = "app"; +/** + * @brief Seed the PRNG with current microseconds and thread ID. + */ +static void rd_kafka_srand(void) { + srand(rd_seed()); +} + void rd_kafka_set_thread_sysname(const char *fmt, ...) { va_list ap; @@ -141,6 +155,31 @@ void rd_kafka_set_thread_sysname(const char *fmt, ...) { thrd_setname(rd_kafka_thread_sysname); } +/** + * @brief Seed the PRNG for the current thread or for the whole process. + * Depending on the platform implementation of srand() the seed can + * be a thread local or global one. In case it's thread local we + * need to call it on each thread. + * + * @param rk Client instance. + * @param internal_thread If true, seed the PRNG if + * it's required per-thread. + */ +#ifdef _WIN32 +void rd_kafka_thread_srand(rd_kafka_t *rk, rd_bool_t internal_thread) { + /* We always call it once per thread in the internal threads and + * once per thread in the application threads, if requested. */ + if (rk->rk_conf.enable_random_seed || internal_thread) + call_once(&rd_kafka_srand_once, rd_kafka_srand); +} +#else +void rd_kafka_thread_srand(rd_kafka_t *rk, rd_bool_t internal_thread) { + /* We call it only in the app threads, only once, if requested. */ + if (rk->rk_conf.enable_random_seed && !internal_thread) + call_once(&rd_kafka_srand_once, rd_kafka_srand); +} +#endif + static void rd_kafka_global_init0(void) { cJSON_Hooks json_hooks = {.malloc_fn = rd_malloc, .free_fn = rd_free}; @@ -171,18 +210,6 @@ void rd_kafka_global_init(void) { } -/** - * @brief Seed the PRNG with current_time.milliseconds - */ -static void rd_kafka_global_srand(void) { - struct timeval tv; - - rd_gettimeofday(&tv, NULL); - - srand((unsigned int)(tv.tv_usec / 1000)); -} - - /** * @returns the current number of active librdkafka instances */ @@ -702,6 +729,9 @@ static const struct rd_kafka_err_desc rd_kafka_err_descs[] = { "Broker: Request principal deserialization failed during " "forwarding"), _ERR_DESC(RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID, "Broker: Unknown topic id"), + _ERR_DESC(RD_KAFKA_RESP_ERR_INCONSISTENT_TOPIC_ID, + "Broker: The log's topic ID did not match the topic ID in the " + "request"), _ERR_DESC(RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH, "Broker: The member epoch is fenced by the group coordinator"), _ERR_DESC(RD_KAFKA_RESP_ERR_UNRELEASED_INSTANCE_ID, @@ -712,15 +742,51 @@ static const struct rd_kafka_err_desc rd_kafka_err_descs[] = { "the consumer group"), _ERR_DESC(RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, "Broker: The member epoch is stale"), + _ERR_DESC(RD_KAFKA_RESP_ERR_MISMATCHED_ENDPOINT_TYPE, + "Broker: The request was sent to an endpoint of the wrong type"), + _ERR_DESC(RD_KAFKA_RESP_ERR_UNSUPPORTED_ENDPOINT_TYPE, + "Broker: This endpoint type is not supported yet"), + _ERR_DESC(RD_KAFKA_RESP_ERR_UNKNOWN_CONTROLLER_ID, + "Broker: This controller ID is not known"), _ERR_DESC(RD_KAFKA_RESP_ERR_UNKNOWN_SUBSCRIPTION_ID, "Broker: Client sent a push telemetry request with an invalid or " "outdated subscription ID"), _ERR_DESC(RD_KAFKA_RESP_ERR_TELEMETRY_TOO_LARGE, "Broker: Client sent a push telemetry request larger than the " "maximum size the broker will accept"), + _ERR_DESC(RD_KAFKA_RESP_ERR_INVALID_REGISTRATION, + "Broker: The controller has considered the broker registration " + "to be invalid"), + _ERR_DESC(RD_KAFKA_RESP_ERR_TRANSACTION_ABORTABLE, + "Broker: The server encountered an error with the transaction"), + _ERR_DESC(RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE, + "Broker: The record state is invalid"), + _ERR_DESC(RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + "Broker: The share session was not found"), + _ERR_DESC(RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH, + "Broker: The share session epoch is invalid"), + _ERR_DESC(RD_KAFKA_RESP_ERR_FENCED_STATE_EPOCH, + "Broker: The share-group state epoch did not match"), + _ERR_DESC(RD_KAFKA_RESP_ERR_INVALID_VOTER_KEY, + "Broker: The voter key doesn't match the receiving replica's " + "key"), + _ERR_DESC(RD_KAFKA_RESP_ERR_DUPLICATE_VOTER, + "Broker: The voter is already part of the set of voters"), + _ERR_DESC(RD_KAFKA_RESP_ERR_VOTER_NOT_FOUND, + "Broker: The voter is not part of the set of voters"), + _ERR_DESC(RD_KAFKA_RESP_ERR_INVALID_REGULAR_EXPRESSION, + "Broker: The regular expression is not valid"), _ERR_DESC(RD_KAFKA_RESP_ERR_REBOOTSTRAP_REQUIRED, "Broker: Client metadata is stale, " - "client should rebootstrap to obtain new metadata."), + "client should rebootstrap to obtain new metadata"), + _ERR_DESC(RD_KAFKA_RESP_ERR_STREAMS_INVALID_TOPOLOGY, + "Broker: The supplied topology is invalid"), + _ERR_DESC(RD_KAFKA_RESP_ERR_STREAMS_INVALID_TOPOLOGY_EPOCH, + "Broker: The supplied topology epoch is invalid"), + _ERR_DESC(RD_KAFKA_RESP_ERR_STREAMS_TOPOLOGY_FENCED, + "Broker: The supplied topology epoch is outdated"), + _ERR_DESC(RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED, + "Broker: The limit of share sessions has been reached"), _ERR_DESC(RD_KAFKA_RESP_ERR__END, NULL)}; @@ -993,18 +1059,37 @@ void rd_kafka_destroy_final(rd_kafka_t *rk) { rd_kafka_assignment_destroy(rk); if (rk->rk_consumer.q) rd_kafka_q_destroy(rk->rk_consumer.q); - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio); - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_current.rk_avg_rebalance_latency); - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_current.rk_avg_commit_latency); - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_poll_idle_ratio); - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_rebalance_latency); - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_commit_latency); + + /* TODO: factor out a helper that destroys both the + * rd_avg_current and rd_avg_rollover variants of a metric + * in one call, to reduce the repetition below. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current + .rk_avg_share_poll_idle_ratio); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_poll_idle_ratio); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current + .rk_avg_share_time_between_poll); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_time_between_poll); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current + .rk_avg_rebalance_latency); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_rebalance_latency); + } else { + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current + .rk_avg_poll_idle_ratio); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_poll_idle_ratio); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current + .rk_avg_rebalance_latency); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current + .rk_avg_commit_latency); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_rebalance_latency); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_commit_latency); + } } /* Purge op-queues */ @@ -1056,6 +1141,7 @@ void rd_kafka_destroy_final(rd_kafka_t *rk) { mtx_destroy(&rk->rk_conf.sasl.lock); rwlock_destroy(&rk->rk_lock); + rd_free(rk->rk_rkshare); rd_free(rk); rd_kafka_global_cnt_decr(); } @@ -1072,6 +1158,10 @@ static void rd_kafka_destroy_app(rd_kafka_t *rk, int flags) { "Terminate", "DestroyCalled", "Immediate", "NoConsumerClose", NULL}; /* Fatal errors and _F_IMMEDIATE also sets .._NO_CONSUMER_CLOSE */ + /* TODO KIP-932: Decide how destroy should behave when a fatal error + * is set: flush pending acks and send the share-session leave + * (as the explicit close() path does), or keep forcing + * NO_CONSUMER_CLOSE and skip those network calls. */ if (flags & RD_KAFKA_DESTROY_F_IMMEDIATE || rd_kafka_fatal_error_code(rk)) flags |= RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE; @@ -1129,7 +1219,26 @@ static void rd_kafka_destroy_app(rd_kafka_t *rk, int flags) { if (rk->rk_cgrp) { rd_kafka_dbg(rk, GENERIC, "TERMINATE", "Terminating consumer group handler"); - rd_kafka_consumer_close(rk); + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_kafka_error_t *error = + rd_kafka_share_consumer_close(rk->rk_rkshare); + if (error) + rd_kafka_error_destroy(error); + } else + rd_kafka_consumer_close(rk); + } + + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + /* Destroy inflight acks map to release + * toppar references held by topic_partition objects in the map. + * Otherwise rd_kafka_destroy_internal() deadlocks: it joins + * broker threads, which wait for refcnt <= 1, but the toppar + * holds a broker ref via rktp_leader that is only released when + * the toppar is destroyed, which requires refcnt 0, which + * requires releasing the rktp ref held by the inflight_acks map + * entry. + */ + RD_MAP_DESTROY(&rk->rk_rkshare->rkshare_inflight_acks); } /* Await telemetry termination. This method blocks until the last @@ -1184,10 +1293,50 @@ void rd_kafka_destroy(rd_kafka_t *rk) { rd_kafka_destroy_app(rk, 0); } +rd_kafka_error_t *rd_kafka_share_destroy(rd_kafka_share_t *rkshare) { + rd_kafka_error_t *error = NULL; + + /** + * TODO KIP-932: Guard this with checks for rkshare and + * rkshare->rkshare_rk? + */ + /* Claim the access gate for this thread so that, from here on, a share + * API call from any *other* thread is rejected for the duration of the + * teardown. Never released: the gate (and its mutex) is destroyed in + * rd_kafka_destroy_app(), and the current_owner_thread/ref_count + * bookkeeping is never read again, so a release would be a no-op. The + * teardown re-enters the gate on this same thread via + * rd_kafka_share_consumer_close(), which the re-entrant ref_count + * permits. A non-NO_ERROR acquire means another + * thread is concurrently inside a share API (or destroy was called from + * the ack callback): destroying now would be unsafe, so report the + * error and leave the instance intact for the application to retry. */ + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + rd_kafka_destroy(rkshare->rkshare_rk); + return NULL; +} + void rd_kafka_destroy_flags(rd_kafka_t *rk, int flags) { rd_kafka_destroy_app(rk, flags); } +rd_kafka_error_t *rd_kafka_share_destroy_flags(rd_kafka_share_t *rkshare, + int flags) { + rd_kafka_error_t *error = NULL; + + /* See rd_kafka_share_destroy(): claim the gate to lock other threads + * out for the teardown. Never released (the gate is destroyed in + * rd_kafka_destroy_app()). On a non-NULL acquire another thread is + * using the consumer; forward the error */ + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + rd_kafka_destroy_flags(rkshare->rkshare_rk, flags); + return NULL; +} + /** * Main destructor for rd_kafka_t @@ -2060,6 +2209,25 @@ static void rd_kafka_1s_tmr_cb(rd_kafka_timers_t *rkts, void *arg) { rd_kafka_coord_cache_expire(&rk->rk_coord_cache); } +/** + * @brief Reset broker down reported flag for all brokers. + * In case it was set to 1 it will be reset to 0 and + * the broker down count will be decremented. + * + * @locks none + * @locks_acquired rd_kafka_rdlock() + * @locality any + */ +void rd_kafka_reset_any_broker_down_reported(rd_kafka_t *rk) { + rd_kafka_broker_t *rkb; + rd_kafka_rdlock(rk); + TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) { + if (rd_atomic32_set(&rkb->rkb_down_reported, 0) == 1) + rd_atomic32_sub(&rk->rk_broker_down_cnt, 1); + } + rd_kafka_rdunlock(rk); +} + /** * @brief Re-bootstrap timer callback. * @@ -2077,18 +2245,19 @@ static void rd_kafka_rebootstrap_tmr_cb(rd_kafka_timers_t *rkts, void *arg) { /* Avoid re-bootstrapping while terminating */ return; + rd_dassert(rk->rk_conf.metadata_recovery_strategy != + RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE); if (rk->rk_conf.metadata_recovery_strategy == - RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE) { - rd_kafka_set_fatal_error( - rk, RD_KAFKA_RESP_ERR_REBOOTSTRAP_REQUIRED, "%s", - "Lost connection to broker(s) " - "and metadata recovery with re-bootstrap " - "is disabled"); + RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE) + /* This function should not be called in this case. + * this is just a fail-safe. */ return; - } rd_kafka_dbg(rk, ALL, "REBOOTSTRAP", "Starting re-bootstrap sequence"); + rd_atomic32_set(&rk->rk_rebootstrap_in_progress, 1); + rd_kafka_reset_any_broker_down_reported(rk); + if (rk->rk_conf.brokerlist) { rd_kafka_brokers_add0( rk, @@ -2100,7 +2269,7 @@ static void rd_kafka_rebootstrap_tmr_cb(rd_kafka_timers_t *rkts, void *arg) { rd_kafka_rdlock(rk); if (rd_list_cnt(&rk->additional_brokerlists) == 0) { rd_kafka_rdunlock(rk); - return; + goto done; } rd_list_init_copy(&additional_brokerlists, &rk->additional_brokerlists); @@ -2115,6 +2284,8 @@ static void rd_kafka_rebootstrap_tmr_cb(rd_kafka_timers_t *rkts, void *arg) { * names even if requested */); } rd_list_destroy(&additional_brokerlists); +done: + rd_atomic32_set(&rk->rk_rebootstrap_in_progress, 0); } static void rd_kafka_stats_emit_tmr_cb(rd_kafka_timers_t *rkts, void *arg) { @@ -2187,6 +2358,28 @@ static int rd_kafka_init_wait(rd_kafka_t *rk, int timeout_ms) { } +/* Forward declarations: defined after reply/fanout handlers */ +static rd_kafka_broker_t *rd_kafka_share_select_broker(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg); + +/** + * @brief Rate-limited log for a stalled share fetch (no progress on the + * main-thread re-trigger path). + * @locality main thread + */ +static void rd_kafka_share_log_fetch_stall(rd_kafka_t *rk, const char *reason) { + rd_kafka_cgrp_t *rkcg = rk->rk_cgrp; + rd_ts_t now = rd_clock(); + + if (rkcg->rkcg_share.fetch_stall_logged_last_ts == 0 || + now > + rkcg->rkcg_share.fetch_stall_logged_last_ts + 5000000 /*5s*/) { + rd_kafka_dbg(rk, CONSUMER, "FETCHMORE", "Fetch stalled: %s", + reason); + rkcg->rkcg_share.fetch_stall_logged_last_ts = now; + } +} + /** * Main loop for Kafka handler thread. */ @@ -2196,6 +2389,7 @@ static int rd_kafka_thread_main(void *arg) { rd_kafka_set_thread_name("main"); rd_kafka_set_thread_sysname("rdk:main"); + rd_kafka_thread_srand(rk, rd_true /* we're in an internal thread */); rd_kafka_interceptors_on_thread_start(rk, RD_KAFKA_THREAD_MAIN); @@ -2242,6 +2436,46 @@ static int rd_kafka_thread_main(void *arg) { RD_KAFKA_Q_CB_CALLBACK, NULL, NULL); if (rk->rk_cgrp) /* FIXME: move to timer-triggered */ rd_kafka_cgrp_serve(rk->rk_cgrp); + + /* KIP-932: If share_fetch_more_records is set but no fetch + * op is in-flight and partitions are assigned, re-trigger + * the fetch flow by selecting a broker directly. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk) && rk->rk_cgrp && + !(rk->rk_cgrp->rkcg_flags & RD_KAFKA_CGRP_F_TERMINATE) && + rk->rk_cgrp->rkcg_share.share_fetch_more_records && + rk->rk_cgrp->rkcg_share + .share_should_fetch_ops_in_flight_cnt == 0) { + if (rd_list_cnt(&rk->rk_cgrp->rkcg_toppars) == 0) { + rd_kafka_share_log_fetch_stall( + rk, + "share_fetch_more_records set but no " + "partitions are assigned"); + } else { + rd_kafka_broker_t *rkb_sel = + rd_kafka_share_select_broker(rk, + rk->rk_cgrp); + if (rkb_sel) { + rd_kafka_dbg( + rk, CONSUMER, "FETCHMORE", + "Re-triggering fetch for " + "share_fetch_more_records=true " + "and no fetch in-flight"); + rd_kafka_share_enqueue_fetch_op( + rk, rkb_sel, rd_true, rd_false); + rd_kafka_broker_destroy(rkb_sel); + rk->rk_cgrp->rkcg_share + .fetch_stall_logged_last_ts = 0; + } else { + rd_kafka_share_log_fetch_stall( + rk, + "no eligible broker (assigned " + "partition leaders are down, " + "terminating, or already have a " + "fetch in flight)"); + } + } + } + rd_kafka_timers_run(&rk->rk_timers, RD_POLL_NOWAIT); } @@ -2345,10 +2579,7 @@ rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, * freed from rd_kafka_destroy_internal() * as the rk itself is destroyed. */ - /* Seed PRNG, don't bother about HAVE_RAND_R, since it is pretty cheap. - */ - if (rk->rk_conf.enable_random_seed) - call_once(&rd_kafka_global_srand_once, rd_kafka_global_srand); + rd_kafka_thread_srand(rk, rd_false /* we're on an app thread */); /* Call on_new() interceptors */ rd_kafka_interceptors_on_new(rk, &rk->rk_conf); @@ -2379,6 +2610,7 @@ rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, rd_atomic32_init(&rk->rk_logical_broker_cnt, 0); rd_atomic32_init(&rk->rk_broker_up_cnt, 0); rd_atomic32_init(&rk->rk_broker_down_cnt, 0); + rd_atomic32_init(&rk->rk_rebootstrap_in_progress, 0); rk->rk_rep = rd_kafka_q_new(rk); rk->rk_ops = rd_kafka_q_new(rk); @@ -2433,11 +2665,36 @@ rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, #if WITH_OAUTHBEARER_OIDC if (rk->rk_conf.sasl.oauthbearer.method == RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC && - !rk->rk_conf.sasl.oauthbearer.token_refresh_cb) - rd_kafka_conf_set_oauthbearer_token_refresh_cb( - &rk->rk_conf, rd_kafka_oidc_token_refresh_cb); + !rk->rk_conf.sasl.oauthbearer.token_refresh_cb) { + /* Use JWT bearer */ + rk->rk_conf.sasl.oauthbearer.builtin_token_refresh_cb = rd_true; + + if (rk->rk_conf.sasl.oauthbearer.metadata_authentication.type == + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AZURE_IMDS) { + rd_kafka_conf_set_oauthbearer_token_refresh_cb( + &rk->rk_conf, + rd_kafka_oidc_token_metadata_azure_imds_refresh_cb); + } else if ( + rk->rk_conf.sasl.oauthbearer.metadata_authentication.type == + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AWS_IAM) { + rd_kafka_conf_set_oauthbearer_token_refresh_cb( + &rk->rk_conf, + rd_kafka_oidc_token_metadata_aws_iam_refresh_cb); + } else if ( + rk->rk_conf.sasl.oauthbearer.grant_type == + RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_CLIENT_CREDENTIALS) { + rd_kafka_conf_set_oauthbearer_token_refresh_cb( + &rk->rk_conf, + rd_kafka_oidc_token_client_credentials_refresh_cb); + } else { + rd_kafka_conf_set_oauthbearer_token_refresh_cb( + &rk->rk_conf, + rd_kafka_oidc_token_jwt_bearer_refresh_cb); + } + } #endif + rk->rk_controllerid = -1; /* Admin client defaults */ @@ -2580,28 +2837,74 @@ rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, rk->rk_consumer.q = rd_kafka_q_keep(rk->rk_rep); } - rd_avg_init( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_poll_idle_ratio, - RD_AVG_GAUGE, 0, 1, 2, rk->rk_conf.enable_metrics_push); - rd_avg_init( - &rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio, - RD_AVG_GAUGE, 0, 1, 2, rk->rk_conf.enable_metrics_push); - rd_avg_init( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_rebalance_latency, - RD_AVG_GAUGE, 0, 500 * 1000, 2, - rk->rk_conf.enable_metrics_push); - rd_avg_init( - &rk->rk_telemetry.rd_avg_current.rk_avg_rebalance_latency, - RD_AVG_GAUGE, 0, 900000 * 1000, 2, - rk->rk_conf.enable_metrics_push); - rd_avg_init( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_commit_latency, - RD_AVG_GAUGE, 0, 500 * 1000, 2, - rk->rk_conf.enable_metrics_push); - rd_avg_init( - &rk->rk_telemetry.rd_avg_current.rk_avg_commit_latency, - RD_AVG_GAUGE, 0, 500 * 1000, 2, - rk->rk_conf.enable_metrics_push); + /* TODO: factor out a helper that initializes both the + * rd_avg_current and rd_avg_rollover variants of a metric + * in one call, to reduce the repetition below. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_avg_init(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_poll_idle_ratio, + RD_AVG_GAUGE, 0, 1000 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_current + .rk_avg_share_poll_idle_ratio, + RD_AVG_GAUGE, 0, 1000 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_time_between_poll, + RD_AVG_GAUGE, 0, 60 * 1000 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_current + .rk_avg_share_time_between_poll, + RD_AVG_GAUGE, 0, 60 * 1000 * 1000, 2, + rk->rk_conf.enable_metrics_push); + /* Rebalance latency is a consumer-group metric + * recorded by the shared cgrp join-state machine + * (rd_kafka_cgrp_set_join_state), which the share + * consumer also drives, so it must be initialized + * here too — otherwise its mutex is left zeroed and + * the rd_avg_add() on reaching the STEADY state + * dereferences an uninitialized lock (crashes on + * Windows, where a zeroed CRITICAL_SECTION is + * invalid). */ + rd_avg_init(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_rebalance_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_current + .rk_avg_rebalance_latency, + RD_AVG_GAUGE, 0, 900000 * 1000, 2, + rk->rk_conf.enable_metrics_push); + + rd_atomic64_init(&rk->rk_telemetry.share_fetch_total, + 0); + rd_atomic64_init( + &rk->rk_telemetry.acknowledgements_send_total, 0); + } else { + rd_avg_init(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_poll_idle_ratio, + RD_AVG_GAUGE, 0, 1, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_current + .rk_avg_poll_idle_ratio, + RD_AVG_GAUGE, 0, 1, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_rebalance_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_current + .rk_avg_rebalance_latency, + RD_AVG_GAUGE, 0, 900000 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_commit_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rk->rk_telemetry.rd_avg_current + .rk_avg_commit_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, + rk->rk_conf.enable_metrics_push); + } } else if (type == RD_KAFKA_PRODUCER) { rk->rk_eos.transactional_id = @@ -2780,12 +3083,17 @@ rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, /* If rk_conf is a struct-copy of the application configuration * we need to avoid rk_conf fields from being freed from * rd_kafka_destroy_internal() since they belong to app_conf. - * However, there are some internal fields, such as interceptors, - * that belong to rk_conf and thus needs to be cleaned up. + * + * NOTE: The interceptors in rk_conf are shallow-copied pointers to + * the same memory as app_conf, so they should NOT be destroyed here. + * They will be destroyed when the user calls rd_kafka_conf_destroy() + * on app_conf. Destroying them here would cause a double-free. + * * Legacy APIs, sigh.. */ if (app_conf) { rd_kafka_assignors_term(rk); - rd_kafka_interceptors_destroy(&rk->rk_conf); + /* Do NOT destroy interceptors here - they belong to app_conf + * and will be freed when app_conf is destroyed by the user. */ memset(&rk->rk_conf, 0, sizeof(rk->rk_conf)); } @@ -2797,151 +3105,1590 @@ rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, return NULL; } -/** - * Schedules a rebootstrap of the cluster immediately. - */ -void rd_kafka_rebootstrap(rd_kafka_t *rk) { - rd_kafka_timer_start_oneshot(&rk->rk_timers, &rk->rebootstrap_tmr, - rd_true /*restart*/, 0, - rd_kafka_rebootstrap_tmr_cb, NULL); +rd_kafka_share_t *rd_kafka_share_consumer_new(rd_kafka_conf_t *conf, + char *errstr, + size_t errstr_size) { + rd_kafka_share_t *rkshare; + rd_kafka_t *rk; + + if (conf == NULL) { + rd_snprintf(errstr, errstr_size, + "rd_kafka_share_consumer_new(): " + "conf argument must not be NULL"); + return NULL; + } + + /* Mark this conf as belonging to a share consumer. The field is + * library-internal (no property table entry); apps must construct + * share consumers via this function. The flag is read by + * rd_kafka_conf_finalize and RD_KAFKA_IS_SHARE_CONSUMER(rk). + * conf_finalize also rejects share-consumer-incompatible + * properties and applies the library-mandatory defaults for the + * rest. */ + conf->share.is_share_consumer = rd_true; + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, errstr_size); + if (!rk) { + /* If rd_kafka_new() failed it will have set the last error + * and filled out errstr, so we don't need to do that here. */ + return NULL; + } + + rd_kafka_log(rk, LOG_WARNING, "SHARECONSUMER", + "The share consumer is in \"Preview\" and is not " + "recommended for production use"); + + rkshare = rd_calloc(1, sizeof(*rkshare)); + rkshare->rkshare_rk = rk; + rkshare->rkshare_unacked_cnt = 0; + rkshare->rkshare_consumer_closing = rd_false; + + /** + * TODO KIP-932: Check for uniform naming structure for all + * share consumer variables. + */ + rk->rk_share_consumer.share_acknowledgement_commit_cb = NULL; + rk->rk_share_consumer.acknowledgement_commit_cb_opaque = NULL; + rk->rk_share_consumer.in_callback = rd_false; + rk->rk_share_consumer.acknowledgement_commit_cb_registered = rd_false; + rd_atomic64_init(&rk->rk_share_consumer.current_owner_thread, + RD_KAFKA_SHARE_NO_CURRENT_THREAD); + rd_atomic32_init(&rk->rk_share_consumer.ref_count, 0); + + /* Set backpointer from rk to rkshare for access in retry handlers */ + rk->rk_rkshare = rkshare; + + /* Inflight acks map keyed by topic name + partition */ + RD_MAP_INIT(&rkshare->rkshare_inflight_acks, 16, + rd_kafka_topic_partition_cmp, rd_kafka_topic_partition_hash, + rd_kafka_topic_partition_destroy_free, + rd_kafka_share_ack_batches_destroy_free); + + return rkshare; } -/** - * Counts usage of the legacy/simple consumer (rd_kafka_consume_start() with - * friends) since it does not have an API for stopping the cgrp we will need to - * sort that out automatically in the background when all consumption - * has stopped. - * - * Returns 0 if a High level consumer is already instantiated - * which means a Simple consumer cannot co-operate with it, else 1. - * - * A rd_kafka_t handle can never migrate from simple to high-level, or - * vice versa, so we dont need a ..consumer_del(). - */ -int rd_kafka_simple_consumer_add(rd_kafka_t *rk) { - if (rd_atomic32_get(&rk->rk_simple_cnt) < 0) - return 0; +rd_kafka_error_t *rd_kafka_share_acquire(rd_kafka_share_t *rkshare) { + rd_kafka_t *rk; + int64_t tid; + + if (unlikely(!rkshare)) + return rd_kafka_error_new(RD_KAFKA_RESP_ERR__STATE, + "Share consumer handle is NULL"); + + rk = rkshare->rkshare_rk; + + /* Claim ownership of the gate. If the gate is unowned, the CAS + * succeeds and this thread becomes the owner; if the gate is + * already ours, the CAS is skipped and we re-enter; if the + * gate is held by a different thread, the CAS fails and the + * call is rejected as concurrent access. */ + tid = (int64_t)thrd_current_id(); + if (rd_atomic64_get(&rk->rk_share_consumer.current_owner_thread) != + tid && + !rd_atomic64_cas(&rk->rk_share_consumer.current_owner_thread, + RD_KAFKA_SHARE_NO_CURRENT_THREAD, tid)) { + int64_t other = rd_atomic64_get( + &rk->rk_share_consumer.current_owner_thread); + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__CONFLICT, + "Share consumer is not safe for multi-threaded " + "access. currentThread(id: %" PRId64 + ") otherThread(id: %" PRId64 ")", + tid, other); + } - return (int)rd_atomic32_add(&rk->rk_simple_cnt, 1); + /* Reject re-entry from inside the user-defined acknowledgement + * commit callback. The gate is either freshly ours or was + * already ours (the callback runs on the owning thread), so + * do not touch ownership/ref_count — leave the outer caller's + * state intact. */ + if (unlikely(rk->rk_share_consumer.in_callback)) + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__STATE, + "Share consumer methods are not accessible from " + "user-defined acknowledgement commit callback"); + + rd_dassert(rd_atomic32_get(&rk->rk_share_consumer.ref_count) >= 0); + rd_atomic32_add(&rk->rk_share_consumer.ref_count, 1); + return NULL; } +void rd_kafka_share_release(rd_kafka_share_t *rkshare) { + rd_kafka_t *rk; + + if (unlikely(!rkshare)) + return; + + rk = rkshare->rkshare_rk; + + /* Decrement the re-entrancy ref_count; when it reaches 0 the + * outermost acquire has been released and the gate is freed + * so a different thread can claim it on a future call. */ + rd_dassert(rd_atomic32_get(&rk->rk_share_consumer.ref_count) > 0); + if (rd_atomic32_sub(&rk->rk_share_consumer.ref_count, 1) == 0) + rd_atomic64_set(&rk->rk_share_consumer.current_owner_thread, + RD_KAFKA_SHARE_NO_CURRENT_THREAD); +} /** - * rktp fetch is split up in these parts: - * * application side: - * * broker side (handled by current leader broker thread for rktp): - * - the fetch state, initial offset, etc. - * - fetching messages, updating fetched offset, etc. - * - offset commits - * - * Communication between the two are: - * app side -> rdkafka main side: rktp_ops - * broker thread -> app side: rktp_fetchq + * @brief Main thread handler for RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER op. * - * There is no shared state between these threads, instead - * state is communicated through the two op queues, and state synchronization - * is performed by version barriers. + * Updates the registration flag on the main thread. No synchronization + * needed because this flag is only accessed from the main thread. * + * @locality main thread */ +static rd_kafka_op_res_t +rd_kafka_share_ack_commit_cb_register_op(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_kafka_op_t *rko) { + if (rk->rk_rkshare) { + rk->rk_share_consumer.acknowledgement_commit_cb_registered = + rko->rko_u.share_ack_commit_cb_register.registered; + rd_kafka_dbg( + rk, CGRP, "SHAREACK", "Share ack callback %sregistered", + rko->rko_u.share_ack_commit_cb_register.registered ? "" + : "un"); + } + return RD_KAFKA_OP_RES_HANDLED; +} -static RD_UNUSED int rd_kafka_consume_start0(rd_kafka_topic_t *rkt, - int32_t partition, - int64_t offset, - rd_kafka_q_t *rkq) { - rd_kafka_toppar_t *rktp; - if (partition < 0) { - rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION, - ESRCH); - return -1; - } +rd_kafka_error_t *rd_kafka_share_set_acknowledgement_commit_cb( + rd_kafka_share_t *rkshare, + void (*share_acknowledgement_commit_cb)( + rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque), + void *opaque) { + rd_kafka_t *rk; + rd_bool_t was_set, now_set; + rd_kafka_error_t *error = NULL; - if (!rd_kafka_simple_consumer_add(rkt->rkt_rk)) { - rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, EINVAL); - return -1; + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + if (unlikely((error = rd_kafka_share_consumer_closed_error(rkshare)) != + NULL)) + goto done; + + rk = rkshare->rkshare_rk; + + /* These fields are owned by app thread only - no lock needed. */ + was_set = + (rk->rk_share_consumer.share_acknowledgement_commit_cb != NULL); + now_set = (share_acknowledgement_commit_cb != NULL); + + rk->rk_share_consumer.share_acknowledgement_commit_cb = + share_acknowledgement_commit_cb; + rk->rk_share_consumer.acknowledgement_commit_cb_opaque = opaque; + + /* Send op to main thread ONLY if registration state transitions + * (set ↔ unset). Changing callback from A→B doesn't need an op + * since the registered flag is already true on the main thread. */ + if (was_set != now_set) { + rd_kafka_op_t *rko = rd_kafka_op_new_cb( + rk, RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER, + rd_kafka_share_ack_commit_cb_register_op); + rko->rko_u.share_ack_commit_cb_register.registered = now_set; + rd_kafka_q_enq(rk->rk_ops, rko); } - rd_kafka_topic_wrlock(rkt); - rktp = rd_kafka_toppar_desired_add(rkt, partition); - rd_kafka_topic_wrunlock(rkt); +done: + rd_kafka_share_release(rkshare); + return error; +} - /* Verify offset */ - if (offset == RD_KAFKA_OFFSET_BEGINNING || - offset == RD_KAFKA_OFFSET_END || - offset <= RD_KAFKA_OFFSET_TAIL_BASE) { - /* logical offsets */ - } else if (offset == RD_KAFKA_OFFSET_STORED) { - /* offset manager */ +/** + * @locality main thread + * @locks none + */ +static rd_kafka_broker_t *rd_kafka_share_select_broker(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg) { + rd_kafka_broker_t *selected_rkb = NULL, *leader = NULL; + rd_list_t *toppars = &rkcg->rkcg_toppars; + int toppar_cnt = rd_list_cnt(toppars); + size_t i; - if (rkt->rkt_conf.offset_store_method == - RD_KAFKA_OFFSET_METHOD_BROKER && - RD_KAFKAP_STR_IS_NULL(rkt->rkt_rk->rk_group_id)) { - /* Broker based offsets require a group id. */ - rd_kafka_toppar_destroy(rktp); - rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, - EINVAL); - return -1; - } + if (toppar_cnt == 0) + return NULL; - } else if (offset < 0) { - rd_kafka_toppar_destroy(rktp); - rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, EINVAL); - return -1; - } + /* Look through all partitions in order, find the first one which + * has a leader. */ + rd_kafka_dbg(rk, CGRP, "SHARE", + "Selecting broker for share fetch from %d assigned " + "partitions, last picked index = %" PRIusz, + toppar_cnt, rkcg->rkcg_share.last_partition_picked); - rd_kafka_toppar_op_fetch_start(rktp, RD_KAFKA_FETCH_POS(offset, -1), - rkq, RD_KAFKA_NO_REPLYQ); + for (i = 0; i < (size_t)toppar_cnt; i++) { + rd_kafka_toppar_t *rktp; + rkcg->rkcg_share.last_partition_picked += 1; + if (rkcg->rkcg_share.last_partition_picked >= + (size_t)toppar_cnt) + rkcg->rkcg_share.last_partition_picked = 0; - rd_kafka_toppar_destroy(rktp); + rktp = rd_list_elem( + toppars, (int)rkcg->rkcg_share.last_partition_picked); + rd_kafka_toppar_keep(rktp); - rd_kafka_set_last_error(0, 0); - return 0; -} + rd_kafka_toppar_lock(rktp); + leader = rktp->rktp_leader; + if (leader) + rd_kafka_broker_keep(leader); + rd_kafka_toppar_unlock(rktp); + /* Criteria to choose a broker: + * 1. It should be the leader of a partition. + * 2. The broker or instance must not be terminating. + * 3. A share-fetch op must not already be enqueued on it. + * 4. The broker must currently be UP. + * + * The rkb_state read is intentionally unlocked: a stale + * UP value at worst costs one stray op (broker thread + * rejects with ERR__STATE), bounded to one per state + * transition. Without this guard, a DOWN leader causes + * an unbounded main<->broker thread bounce loop. */ + if (leader) { + if (!rd_kafka_broker_or_instance_terminating(leader) && + !leader->rkb_share_fetch_enqueued && + leader->rkb_state == RD_KAFKA_BROKER_STATE_UP) { + rd_kafka_broker_keep(leader); + selected_rkb = leader; + rd_kafka_dbg( + rk, CGRP, "SHARE", + "Selected broker %s (%p) for share " + "fetch " + "for partition %s [%" PRId32 "]", + rd_kafka_broker_name(selected_rkb), + selected_rkb, + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition); + } + /* Release the snapshot reference taken under rktp lock + * above. */ + rd_kafka_broker_destroy(leader); + } + rd_kafka_toppar_destroy(rktp); -int rd_kafka_consume_start(rd_kafka_topic_t *app_rkt, - int32_t partition, - int64_t offset) { - rd_kafka_topic_t *rkt = rd_kafka_topic_proper(app_rkt); - rd_kafka_dbg(rkt->rkt_rk, TOPIC, "START", - "Start consuming partition %" PRId32, partition); - return rd_kafka_consume_start0(rkt, partition, offset, NULL); + if (selected_rkb) + break; + } + return selected_rkb; } -int rd_kafka_consume_start_queue(rd_kafka_topic_t *app_rkt, - int32_t partition, - int64_t offset, - rd_kafka_queue_t *rkqu) { - rd_kafka_topic_t *rkt = rd_kafka_topic_proper(app_rkt); - return rd_kafka_consume_start0(rkt, partition, offset, rkqu->rkqu_q); +/** + * @brief Enqueue a SHARE_FETCH_FANOUT op and set the fetch guard flag + * if the op requests more records. + * @locality app thread + */ +static void rd_kafka_share_fetch_fanout_enqueue(rd_kafka_t *rk, + rd_kafka_op_t *rko) { + if (rko->rko_u.share_fetch_fanout.fetch_more_records) + rk->rk_rkshare->rkshare_fetch_more_records_requested = rd_true; + rd_kafka_q_enq(rk->rk_ops, rko); } +/** + * @brief Enqueue a SHARE_FETCH_FANOUT op on the main queue. + * @param backoff_ms If >0 the op will be enqueued after this many milliseconds. + * Else, it will be immediate. + * @locality app thread + */ +static void rd_kafka_share_fetch_fanout(rd_kafka_t *rk, + rd_bool_t fetch_more_records, + rd_list_t *ack_batches) { + rd_kafka_op_t *rko = rd_kafka_op_new_cb( + rk, RD_KAFKA_OP_SHARE_FETCH_FANOUT, rd_kafka_share_fetch_fanout_op); + rko->rko_u.share_fetch_fanout.fetch_more_records = fetch_more_records; + rko->rko_replyq = RD_KAFKA_REPLYQ(rk->rk_ops, 0); + + /* Attach pre-built ack_details (ownership transferred to op) */ + rko->rko_u.share_fetch_fanout.ack_batches = ack_batches; + + rd_kafka_share_fetch_fanout_enqueue(rk, rko); +} -static RD_UNUSED int rd_kafka_consume_stop0(rd_kafka_toppar_t *rktp) { - rd_kafka_q_t *tmpq = NULL; - rd_kafka_resp_err_t err; +static void rd_kafka_share_enqueue_sync_ack_op(rd_kafka_t *rk, + rd_kafka_broker_t *rkb); +rd_kafka_op_res_t rd_kafka_share_fetch_reply_op(rd_kafka_t *rk, + rd_kafka_op_t *rko_orig); - rd_kafka_topic_wrlock(rktp->rktp_rkt); - rd_kafka_toppar_lock(rktp); - rd_kafka_toppar_desired_del(rktp); - rd_kafka_toppar_unlock(rktp); - rd_kafka_topic_wrunlock(rktp->rktp_rkt); +/** + * @brief rko_op_cb wrapper for rd_kafka_share_fetch_reply_op. + * + * Using rko_op_cb causes rd_kafka_op_reply() to stamp RD_KAFKA_OP_CB + * instead of RD_KAFKA_OP_REPLY, which bypasses the handle_std + * __DESTROY silent-discard path (rdkafka_op.c:987-991). + * + * @locality main thread + */ +static rd_kafka_op_res_t rd_kafka_share_fetch_reply_op_cb(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_kafka_op_t *rko) { + return rd_kafka_share_fetch_reply_op(rk, rko); +} - tmpq = rd_kafka_q_new(rktp->rktp_rkt->rkt_rk); +/** + * Handles RD_KAFKA_OP_SHARE_FETCH | RD_KAFKA_OP_REPLY. + * @locality main thread + */ +rd_kafka_op_res_t rd_kafka_share_fetch_reply_op(rd_kafka_t *rk, + rd_kafka_op_t *rko_orig) { + rd_kafka_broker_t *reply_rkb = + rko_orig->rko_u.share_fetch.target_broker; + rd_kafka_cgrp_t *rkcg = rd_kafka_cgrp_get(rk); + rd_bool_t should_fetch = rko_orig->rko_u.share_fetch.should_fetch; + rd_bool_t records_fetched = rko_orig->rko_u.share_fetch.records_fetched; + rd_bool_t should_leave = rko_orig->rko_u.share_fetch.should_leave; + rd_list_t *ack_details = rko_orig->rko_u.share_fetch.ack_details; + + /* The reply handler is expected to be invoked on the main thread, + * unless the client is terminating with NO_CONSUMER_CLOSE in which + * case it can be invoked on the replying's broker thread + * (main thread would have come out of its event loop by the time broker + * replies) + */ + rd_dassert(thrd_is_current(rk->rk_thread) || + rko_orig->rko_err == RD_KAFKA_RESP_ERR__DESTROY); + rd_kafka_dbg(rk, CGRP, "SHAREFETCH", + "Share fetch reply: %s, should_fetch=%d, " + "records_fetched=%d, should_leave=%d, broker=%s", + rd_kafka_err2str(rko_orig->rko_err), should_fetch, + records_fetched, should_leave, + reply_rkb ? rd_kafka_broker_name(reply_rkb) : "none"); + + + /* Per-partition errors are carried on each batch's + * rktpar->err. The broker thread sets these on both + * success (per-partition error from response via parser) + * and top-level error (top-level err on each batch via + * rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err + * helper). + * + * Defensive: if the helper was bypassed (e.g. q_enq on + * a disabled rkb_ops queue at rdkafka_queue.h:440 falls + * through to rd_kafka_op_reply directly) the batch err + * is still _IN_PROGRESS from build_ack_details. Override + * with rko_err so the sentinel doesn't leak to the + * app. _IN_PROGRESS check is sufficient because the + * normal helper path overwrites _IN_PROGRESS first + * (buf callback init -> INVALID_RECORD_STATE -> per- + * partition err or top-level err). */ + if (ack_details && rko_orig->rko_err) { + rd_kafka_share_ack_batches_t *batch; + int k; + RD_LIST_FOREACH(batch, ack_details, k) { + if (batch->rktpar->err == + RD_KAFKA_RESP_ERR__IN_PROGRESS) + batch->rktpar->err = rko_orig->rko_err; + } + } - rd_kafka_toppar_op_fetch_stop(rktp, RD_KAFKA_REPLYQ(tmpq, 0)); + /* + * Step 1: Dispatch acknowledgement callbacks. + * Per-partition errors were set by the broker thread. + */ + rd_kafka_share_dispatch_ack_callbacks( + rk, rko_orig->rko_u.share_fetch.ack_details); - /* Synchronisation: Wait for stop reply from broker thread */ - err = rd_kafka_q_wait_result(tmpq, RD_POLL_INFINITE); - rd_kafka_q_destroy_owner(tmpq); + reply_rkb->rkb_share_fetch_enqueued = rd_false; - rd_kafka_set_last_error(err, err ? EINVAL : 0); + if (should_fetch) + rkcg->rkcg_share.share_should_fetch_ops_in_flight_cnt--; + + if (should_leave) { + rkcg->rkcg_share.share_session_leave_remaining_cnt--; + return RD_KAFKA_OP_RES_HANDLED; + } + + /* + * Step 2: If records were fetched and broker is not terminating, + * reset the global fetch guard so the next FANOUT can select + * a new fetch broker. + */ + if (records_fetched && rko_orig->rko_err != RD_KAFKA_RESP_ERR__DESTROY) + rkcg->rkcg_share.share_fetch_more_records = rd_false; + + /* + * Step 3: If the consumer has been marked for termination, + * enqueue a session leave op on the replying broker thread and return + */ + if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_TERMINATE) { + if (!rd_kafka_destroy_flags_no_consumer_close(rkcg->rkcg_rk)) + rd_kafka_share_enqueue_fetch_op( + rkcg->rkcg_rk, reply_rkb, rd_false, rd_true); + return RD_KAFKA_OP_RES_HANDLED; + } + + /* + * Step 4: Handle commit_sync reply if this op belongs to + * the current commit_sync request. + */ + if (rko_orig->rko_u.share_fetch.commit_sync_request_id != 0 && + rko_orig->rko_u.share_fetch.commit_sync_request_id == + rkcg->rkcg_commit_sync_request.id) { + rd_kafka_dbg( + rk, CGRP, "SHARE", + "Commit sync reply from broker %s: %s, " + "remaining brokers: %d", + rd_kafka_broker_name(reply_rkb), + rd_kafka_err2str(rko_orig->rko_err), + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt); + rd_kafka_share_commit_sync_apply_result(rk, rkcg, ack_details); + } + + /* + * Step 5: Dispatch pending commit_sync to the replying broker + * (highest priority). Acks must always be sent. + */ + if (!reply_rkb->rkb_share_fetch_enqueued && + !rd_kafka_broker_or_instance_terminating(reply_rkb) && + reply_rkb->rkb_pending_commit_sync.sync_ack_details) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Dispatching pending sync acks to broker %s", + rd_kafka_broker_name(reply_rkb)); + rd_kafka_share_enqueue_sync_ack_op(rk, reply_rkb); + } + + /* + * Step 6: If this was the fetch broker but no records were received, + * try to select another broker to fetch from. + * + * TODO: KIP-932: Handle the case where all assignments are removed + * while a fetch is in progress. In that scenario, no records are + * returned and rd_kafka_share_select_broker() will always return + * NULL (no assigned partitions), leaving fetch_more_records stuck + * at true indefinitely. Need to detect this (e.g. check if + * assignments are empty) and reset fetch_more_records. + */ + if (should_fetch && !records_fetched && + rkcg->rkcg_share.share_fetch_more_records && + !rd_kafka_terminating(rk)) { + rd_kafka_broker_t *selected_rkb; + + selected_rkb = rd_kafka_share_select_broker(rk, rkcg); + if (selected_rkb) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Re-selecting broker %s for share fetch " + "(previous broker %s returned no records)", + rd_kafka_broker_name(selected_rkb), + rd_kafka_broker_name(reply_rkb)); + + rd_kafka_share_enqueue_fetch_op(rk, selected_rkb, + rd_true, rd_false); + rd_kafka_broker_destroy(selected_rkb); + } else { + rd_kafka_dbg(rk, CGRP, "SHARE", + "No broker available for share fetch " + "retry, will retry on next FANOUT"); + } + } + + /* + * Step 7: If the replying broker has cached async ack details, + * send an ack-only SHARE_FETCH op to it. + */ + if (reply_rkb->rkb_share_async_ack_details && + !reply_rkb->rkb_share_fetch_enqueued && + !rd_kafka_broker_or_instance_terminating(reply_rkb)) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Enqueuing ack-only SHARE_FETCH to broker %s " + "to flush pending acks", + rd_kafka_broker_name(reply_rkb)); + rd_kafka_share_enqueue_fetch_op(rk, reply_rkb, rd_false, + rd_false); + } + + return RD_KAFKA_OP_RES_HANDLED; +} + + +/** + * @brief Create and enqueue a SHARE_FETCH op on a broker. + * + * Moves any cached ack details from rkb_share_async_ack_details + * into the op, sets rkb_share_fetch_enqueued, and enqueues the op + * on the broker's ops queue. + * + * @param rk Client instance + * @param rkb Broker to enqueue on. Must not have rkb_share_fetch_enqueued set. + * @param should_fetch Whether the broker should fetch records. + * @param should_leave Whether the broker should close the session. + * + * @locality main thread + */ +void rd_kafka_share_enqueue_fetch_op(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_bool_t should_fetch, + rd_bool_t should_leave) { + rd_kafka_op_t *rko_sf; + + rd_assert(!(should_fetch && should_leave)); + rko_sf = rd_kafka_op_new(RD_KAFKA_OP_SHARE_FETCH); + rko_sf->rko_u.share_fetch.should_leave = should_leave; + rko_sf->rko_u.share_fetch.abs_timeout = + rd_timeout_init(rk->rk_conf.socket_timeout_ms); + rko_sf->rko_u.share_fetch.should_fetch = should_fetch; + + /* Move ack details from broker cache to op */ + rko_sf->rko_u.share_fetch.ack_details = + rkb->rkb_share_async_ack_details; + rkb->rkb_share_async_ack_details = NULL; + + rd_kafka_broker_keep(rkb); + rko_sf->rko_u.share_fetch.target_broker = rkb; + rko_sf->rko_replyq = RD_KAFKA_REPLYQ(rk->rk_ops, 0); + rko_sf->rko_op_cb = rd_kafka_share_fetch_reply_op_cb; + rko_sf->rko_rk = rk; + + /* Set fetch guard flag to prevent multiple in-flight fetches to the + * same broker.*/ + rd_assert(!rkb->rkb_share_fetch_enqueued); + rkb->rkb_share_fetch_enqueued = rd_true; + + if (should_fetch) + rd_kafka_cgrp_get(rk) + ->rkcg_share.share_should_fetch_ops_in_flight_cnt++; + + rd_kafka_dbg(rk, CGRP, "SHAREFETCH", + "Enqueuing share fetch op on broker %s " + "(%s fetch, %s leave, %s acks)", + rd_kafka_broker_name(rkb), should_fetch ? "with" : "no", + should_leave ? "with" : "no", + rko_sf->rko_u.share_fetch.ack_details ? "with" : "no"); + + rd_kafka_q_enq(rkb->rkb_ops, rko_sf); +} + +/** + * @brief Segregate ack batches by partition leader and dispatch + * SHARE_FETCH ops to brokers that have pending acks or are the + * selected fetch broker. + * + * Common logic shared by both SHARE_FETCH_FANOUT and + * SHARE_COMMIT_ASYNC_FANOUT handlers. + * + * @param rk Client instance. + * @param ack_batches List of ack batches to segregate and dispatch. + * Ownership is transferred: the list is destroyed + * after segregation. May be NULL if no acks. + * @param selected_rkb The broker selected for fetching, or NULL for + * ack-only dispatch. If non-NULL, that broker's + * op will have should_fetch=true. + * + * @locality main thread + */ +static void +rd_kafka_share_segregate_and_dispatch_acks(rd_kafka_t *rk, + rd_list_t *ack_batches, + rd_kafka_broker_t *selected_rkb) { + rd_kafka_broker_t *rkb; + + if (ack_batches) { + rd_kafka_share_segregate_acks_by_leader(rk, ack_batches); + + /* Ownership of elements transferred to broker ack_details. + * Destroy the container. */ + rd_list_destroy(ack_batches); + } + + /** + * TODO KIP-932: Instead of iterating through all brokers every time, + * we could keep track of which brokers have pending acks or is + * selected for fetch in a separate list, and only iterate through + * that list here. + */ + rd_kafka_rdlock(rk); + TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) { + rd_bool_t has_acks = (rkb->rkb_share_async_ack_details != NULL); + rd_bool_t is_fetch_broker = (rkb == selected_rkb); + + if (rd_kafka_broker_or_instance_terminating(rkb) || + RD_KAFKA_BROKER_IS_LOGICAL(rkb)) + continue; + + /* Skip brokers with nothing to do */ + if (!has_acks && !is_fetch_broker) + continue; + + /* If a request is already in-flight on this broker, + * acks stay cached in rkb_share_async_ack_details + * and will be sent with the next request. */ + if (rkb->rkb_share_fetch_enqueued) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Broker %s has in-flight request, " + "with acks: %s, " + "is fetch broker: %s", + rd_kafka_broker_name(rkb), + has_acks ? "yes" : "no", + is_fetch_broker ? "yes" : "no"); + continue; + } + + rd_kafka_share_enqueue_fetch_op(rk, rkb, is_fetch_broker, + rd_false); + } + rd_kafka_rdunlock(rk); +} + +/** + * Op callback for RD_KAFKA_OP_SHARE_FETCH_FANOUT. + * @locality main thread + */ +rd_kafka_op_res_t rd_kafka_share_fetch_fanout_op(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_kafka_op_t *rko) { + rd_kafka_broker_t *selected_rkb = NULL; + rd_kafka_cgrp_t *rkcg = rd_kafka_cgrp_get(rk); + rd_bool_t fetch_more_records = + rko->rko_u.share_fetch_fanout.fetch_more_records; + rd_bool_t has_fanout_acks = + (rko->rko_u.share_fetch_fanout.ack_batches && + rd_list_cnt(rko->rko_u.share_fetch_fanout.ack_batches) > 0); + + /* + * Step 1: Global fetch guard — select a fetch broker. + * Select if: + * (a) fetch requested and no fetch is already in progress, OR + * (b) fetch_more_records is already set but no broker actually + * has a fetch enqueued (stuck state from failed re-selection). + */ + if (fetch_more_records) { + rd_bool_t need_select = rd_false; + + if (!rkcg->rkcg_share.share_fetch_more_records) { + rkcg->rkcg_share.share_fetch_more_records = rd_true; + need_select = rd_true; + } else if (rkcg->rkcg_share + .share_should_fetch_ops_in_flight_cnt == 0) { + need_select = rd_true; + } + + if (need_select) { + selected_rkb = rd_kafka_share_select_broker(rk, rkcg); + + if (selected_rkb) + rd_kafka_dbg( + rk, CGRP, "SHARE", + "Selected broker %s for fetching " + "messages", + rd_kafka_broker_name(selected_rkb)); + else + rd_kafka_dbg( + rk, CGRP, "SHARE", + "No broker available for share fetch"); + } + } + + if (!selected_rkb && !has_fanout_acks) { + rd_kafka_dbg(rk, CGRP, "SHARE", "No fetch or acks to fan out"); + return RD_KAFKA_OP_RES_HANDLED; + } + + /* + * Step 2: Segregate acks by partition leader and dispatch + * SHARE_FETCH ops to brokers with pending acks or the selected + * fetch broker. + */ + rd_kafka_share_segregate_and_dispatch_acks( + rk, + has_fanout_acks ? rko->rko_u.share_fetch_fanout.ack_batches : NULL, + selected_rkb); + rko->rko_u.share_fetch_fanout.ack_batches = NULL; + + RD_IF_FREE(selected_rkb, rd_kafka_broker_destroy); + + return RD_KAFKA_OP_RES_HANDLED; +} + +/** + * @brief Return an error if the share consumer is closed or closing. + * + * @param rkshare Share consumer instance. + * @returns error if closed/closing, NULL otherwise. Caller owns the error. + */ +rd_kafka_error_t * +rd_kafka_share_consumer_closed_error(rd_kafka_share_t *rkshare) { + if (unlikely(rd_kafka_share_consumer_closed(rkshare) || + rkshare->rkshare_consumer_closing)) + return rd_kafka_error_new(RD_KAFKA_RESP_ERR__STATE, + "Share consumer is closed"); + return NULL; +} + +rd_kafka_resp_err_t +rd_kafka_share_consumer_closed_err(rd_kafka_share_t *rkshare) { + rd_kafka_error_t *error; + rd_kafka_resp_err_t err; + + error = rd_kafka_share_consumer_closed_error(rkshare); + if (!error) + return RD_KAFKA_RESP_ERR_NO_ERROR; + + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + return err; +} + +/** + * @brief Record the start of a share consume batch. + * + * Enforces max.poll.interval.ms for the share consumer. + * While blocked waiting for messages, sets rk_ts_last_poll to + * INT64_MAX so the wait isn't counted against the interval. + * Records the time-between-poll telemetry sample. + */ +static void rd_kafka_share_record_poll_start(rd_kafka_t *rk, int timeout_ms) { + rd_ts_t now; + + /* max.poll.interval.ms enforcement: always mark the consumer as + * blocked inside librdkafka for the duration of the batch. */ + rd_atomic64_set(&rk->rk_ts_last_poll, INT64_MAX); + + /* The remaining work is telemetry sampling only. */ + if (!rk->rk_conf.enable_metrics_push) + return; + + now = rd_clock(); + rk->rk_telemetry.rk_share_poll.ts_poll_start = now; + + /* First poll has no previous timestamp to compare against, so the + * first sample contributes 0 to the average. */ + if (rk->rk_telemetry.rk_share_poll.ts_last_poll_start) + rk->rk_telemetry.rk_share_poll.time_since_last_poll = + now - rk->rk_telemetry.rk_share_poll.ts_last_poll_start; + else + rk->rk_telemetry.rk_share_poll.time_since_last_poll = 0; + + /* Store in microseconds; calculator scales to ms at read time. + * Storing in ms here would truncate sub-millisecond samples to 0 + * (e.g., 999 μs / 1000 = 0), under-reporting at high poll rates. */ + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_time_between_poll, + rk->rk_telemetry.rk_share_poll.time_since_last_poll); + rk->rk_telemetry.rk_share_poll.ts_last_poll_start = now; +} + +/** + * @brief Record the end of a share consume batch. + * + * Enforces max.poll.interval.ms for the share consumer. + * Records the last poll time and expedites the next heartbeat if the + * interval had been exceeded. Records the poll idle ratio sample + * (pollTime / (pollTime + timeSinceLastPoll), mirroring Java's formula). + */ +static void rd_kafka_share_record_poll_end(rd_kafka_t *rk) { + rd_ts_t end, poll_time, total; + + end = rd_clock(); + + /* max.poll.interval.ms enforcement: record the poll time and, if the + * interval had been exceeded, expedite the next heartbeat to rejoin. */ + rd_atomic64_set(&rk->rk_ts_last_poll, end); + if (unlikely(rk->rk_cgrp && rk->rk_cgrp->rkcg_flags & + RD_KAFKA_CGRP_F_MAX_POLL_EXCEEDED)) + rd_kafka_cgrp_consumer_expedite_next_heartbeat( + rk->rk_cgrp, "app polled after poll interval exceeded"); + + /* The remaining work is telemetry sampling only. */ + if (!rk->rk_conf.enable_metrics_push) + return; + + poll_time = end - rk->rk_telemetry.rk_share_poll.ts_poll_start; + total = poll_time + rk->rk_telemetry.rk_share_poll.time_since_last_poll; + if (total > 0) { + int64_t poll_idle_ratio = poll_time * 1000000 / total; + rd_avg_add(&rk->rk_telemetry.rd_avg_current + .rk_avg_share_poll_idle_ratio, + poll_idle_ratio); + } +} + +size_t rd_kafka_messages_count(const rd_kafka_messages_t *messages) { + if (!messages) + return 0; + return messages->cnt; +} + +rd_kafka_message_t *rd_kafka_messages_get(const rd_kafka_messages_t *messages, + size_t index) { + return messages->elems[index]; +} + +void rd_kafka_messages_destroy(rd_kafka_messages_t *messages) { + size_t i; + if (!messages) + return; + for (i = 0; i < messages->cnt; i++) { + rd_kafka_message_destroy(messages->elems[i]); + } + rd_free(messages); +} + + +rd_kafka_error_t *rd_kafka_share_poll(rd_kafka_share_t *rkshare, + int timeout_ms, + rd_kafka_messages_t **rkmessages) { + rd_kafka_t *rk = rkshare->rkshare_rk; + rd_kafka_cgrp_t *rkcg; + rd_bool_t has_records; + rd_bool_t has_pending_acks; + rd_kafka_error_t *error = NULL; + rd_list_t *ack_batches = NULL; + rd_bool_t need_fetch_more_records; + + /* Always NULL the out param so error/empty paths are well-defined. */ + *rkmessages = NULL; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + /* TODO KIP-932: the non-fatal errors returned from the other paths + * below (consumer-group-not-initialized, consumer-closed, and the + * not-all-acknowledged guard) should be marked retriable so the app + * knows it can retry the share_poll call, consistent with the + * retriable errors built in rd_kafka_q_serve_share_rkmessages(). */ + if (unlikely(!(rkcg = rd_kafka_cgrp_get(rk)))) { + error = rd_kafka_error_new(RD_KAFKA_RESP_ERR__STATE, + "rd_kafka_share_poll(): " + "Consumer group not initialized"); + goto done; + } + + if (unlikely((error = rd_kafka_share_consumer_closed_error(rkshare)) != + NULL)) + goto done; + + /* Drain rk_rep for all pending callbacks (non-blocking) */ + rd_kafka_q_serve(rk->rk_rep, RD_POLL_NOWAIT, 0, RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); + + /* In implicit ack mode, convert all ACQUIRED records to ACCEPT + * before extracting ack details. In explicit mode, the app has + * already acknowledged records via the acknowledge APIs, so + * only extract what's been explicitly acknowledged. */ + rd_kafka_share_acknowledge_all_if_implicit(rkshare); + + error = rd_kafka_share_ensure_all_acknowledged_if_explicit(rkshare); + if (error) + goto done; + + /* Fail early if no subscription is active. + * rkshare_subscribed is app-thread-owned and set/cleared by + * rd_kafka_share_subscribe / rd_kafka_share_unsubscribe under + * the same rkshare_acquire/release lock that guards this call. */ + if (unlikely(!rkshare->rkshare_subscribed)) { + error = rd_kafka_error_new( + RD_KAFKA_RESP_ERR__STATE, + "Consumer is not subscribed to any topics"); + goto done; + } + + rd_kafka_share_record_poll_start(rk, timeout_ms); + + ack_batches = rd_kafka_share_build_ack_details(rk->rk_rkshare); + + has_records = rd_kafka_q_len(rkcg->rkcg_q) > 0; + has_pending_acks = ack_batches != NULL; + + /* Only request fetch if no fetch FANOUT is already in flight */ + need_fetch_more_records = + !has_records && + !rk->rk_rkshare->rkshare_fetch_more_records_requested; + + if (need_fetch_more_records || has_pending_acks) + rd_kafka_share_fetch_fanout(rk, need_fetch_more_records, + ack_batches); + + error = rd_kafka_q_serve_share_rkmessages(rkcg->rkcg_q, timeout_ms, + rkmessages); + + /* Drain rk_rep for callbacks again before returning */ + rd_kafka_q_serve(rk->rk_rep, RD_POLL_NOWAIT, 0, RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); + + rd_kafka_share_record_poll_end(rk); + +done: + rd_kafka_share_release(rkshare); + return error; +} + +/** + * @brief Main thread handler for SHARE_COMMIT_ASYNC_FANOUT op. + * + * Segregates acks by partition leader and sends ack-only + * SHARE_FETCH ops to the respective brokers. + * + * @locality main thread + */ +rd_kafka_op_res_t rd_kafka_share_commit_async_fanout_op(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_kafka_op_t *rko) { + rd_bool_t has_acks = + (rko->rko_u.share_commit_async_fanout.ack_batches && + rd_list_cnt(rko->rko_u.share_commit_async_fanout.ack_batches) > 0); + + if (!has_acks) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "No acks to commit in " + "SHARE_COMMIT_ASYNC_FANOUT"); + return RD_KAFKA_OP_RES_HANDLED; + } + + rd_kafka_share_segregate_and_dispatch_acks( + rk, rko->rko_u.share_commit_async_fanout.ack_batches, + NULL /* ack-only, no fetch */); + rko->rko_u.share_commit_async_fanout.ack_batches = NULL; + + return RD_KAFKA_OP_RES_HANDLED; +} + +/** + * @brief Merge ack batches from \p src_list into \p dst_list. + * + * For each batch in src_list, finds a matching topic-partition in + * dst_list. If found, deep-copies entries from src into existing + * batch and re-sorts by start_offset. If not found, moves the + * batch as-is. + * + * @param dst_list Destination list (rd_kafka_share_ack_batches_t*). + * @param src_list Source list (rd_kafka_share_ack_batches_t*). + * Elements are moved or destroyed; list is emptied. + * + * @locality main thread + */ +static void rd_kafka_share_merge_ack_lists(rd_list_t *dst_list, + rd_list_t *src_list) { + rd_kafka_share_ack_batches_t *batch; + + while ((batch = rd_list_pop(src_list))) { + rd_kafka_share_ack_batches_t *existing; + + existing = + rd_kafka_share_find_ack_batch(dst_list, batch->rktpar); + if (existing) { + rd_kafka_share_ack_batch_entry_t *entry; + int j; + RD_LIST_FOREACH(entry, &batch->entries, j) { + rd_list_add( + &existing->entries, + rd_kafka_share_ack_batch_entry_copy(entry)); + } + rd_list_sort(&existing->entries, + rd_kafka_share_ack_entries_sort_cmp_ptr); + rd_kafka_share_ack_batches_destroy(batch); + } else { + rd_list_add(dst_list, batch); + } + } +} + +/** + * @brief Timeout callback for commit_sync. + * + * Fires when the commit_sync timeout expires. For each broker with + * pending_commit_sync, merges those acks into async_ack_details so + * they are sent asynchronously later. Fills all partitions still + * marked as IN_PROGRESS with REQUEST_TIMED_OUT. Sends the response. + * + * @locality main thread (timer thread -> main thread) + */ +static void rd_kafka_share_commit_sync_timeout_cb(rd_kafka_timers_t *rkts, + void *arg) { + rd_kafka_t *rk = rkts->rkts_rk; + rd_kafka_cgrp_t *rkcg = rd_kafka_cgrp_get(rk); + rd_kafka_broker_t *rkb; + rd_kafka_topic_partition_list_t *results; + int i; + + rd_kafka_dbg( + rk, CGRP, "SHARE", + "Commit sync timeout fired, " + "pending brokers: %d", + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt); + + /* Move pending_commit_sync acks to async_ack_details */ + rd_kafka_rdlock(rk); + TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) { + if (!rkb->rkb_pending_commit_sync.sync_ack_details) + continue; + + if (!rkb->rkb_share_async_ack_details) + rkb->rkb_share_async_ack_details = rd_list_new( + rd_list_cnt( + rkb->rkb_pending_commit_sync.sync_ack_details), + rd_kafka_share_ack_batches_destroy_free); + + rd_kafka_share_merge_ack_lists( + rkb->rkb_share_async_ack_details, + rkb->rkb_pending_commit_sync.sync_ack_details); + + rd_list_destroy(rkb->rkb_pending_commit_sync.sync_ack_details); + rkb->rkb_pending_commit_sync.sync_ack_details = NULL; + rkb->rkb_pending_commit_sync.abs_timeout = 0; + rkb->rkb_pending_commit_sync.commit_sync_request_id = 0; + } + rd_kafka_rdunlock(rk); + + /* TODO: Verify that __TIMED_OUT (local timeout) is the correct + * error here rather than REQUEST_TIMED_OUT (broker error). */ + /* Fill partitions still IN_PROGRESS with REQUEST_TIMED_OUT + * to surface the commit_sync deadline expiry to the caller. */ + results = rkcg->rkcg_commit_sync_request.results; + for (i = 0; i < results->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &results->elems[i]; + if (rktpar->err == RD_KAFKA_RESP_ERR__IN_PROGRESS) + rktpar->err = RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + } + + rd_kafka_share_commit_sync_send_response(rkcg); +} + +/** + * @brief Create and enqueue a sync ack-only SHARE_FETCH op on a broker. + * + * Moves ack details, abs_timeout, and commit_sync_request_id from + * rkb_pending_commit_sync into the op. + * + * @param rk Client instance. + * @param rkb Broker to enqueue on. Must have pending_commit_sync data. + * + * @locality main thread + */ +static void rd_kafka_share_enqueue_sync_ack_op(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_kafka_op_t *rko_sf; + + rko_sf = rd_kafka_op_new(RD_KAFKA_OP_SHARE_FETCH); + rko_sf->rko_u.share_fetch.should_leave = rd_false; + rko_sf->rko_u.share_fetch.abs_timeout = + rkb->rkb_pending_commit_sync.abs_timeout; + rko_sf->rko_u.share_fetch.should_fetch = rd_false; + rko_sf->rko_u.share_fetch.commit_sync_request_id = + rkb->rkb_pending_commit_sync.commit_sync_request_id; + + /* Move ack details from pending_commit_sync to op */ + rko_sf->rko_u.share_fetch.ack_details = + rkb->rkb_pending_commit_sync.sync_ack_details; + rkb->rkb_pending_commit_sync.sync_ack_details = NULL; + rkb->rkb_pending_commit_sync.abs_timeout = 0; + rkb->rkb_pending_commit_sync.commit_sync_request_id = 0; + + rd_kafka_broker_keep(rkb); + rko_sf->rko_u.share_fetch.target_broker = rkb; + rko_sf->rko_replyq = RD_KAFKA_REPLYQ(rk->rk_ops, 0); + rko_sf->rko_op_cb = rd_kafka_share_fetch_reply_op_cb; + rko_sf->rko_rk = rk; + + rkb->rkb_share_fetch_enqueued = rd_true; + + rd_kafka_dbg( + rk, CGRP, "SHARE", + "Enqueuing sync ack op on broker %s " + "(abs_timeout %" PRId64 ", commit_sync_request_id %" PRId64 ")", + rd_kafka_broker_name(rkb), rko_sf->rko_u.share_fetch.abs_timeout, + rko_sf->rko_u.share_fetch.commit_sync_request_id); + + rd_kafka_q_enq(rkb->rkb_ops, rko_sf); +} + +/** + * @brief Dispatch pending sync ack requests to free brokers. + * + * Phase 2 of sync dispatch: iterates all brokers. For each broker + * that has pending_commit_sync, increments brokers_awaiting_result_cnt. + * If the broker is free (no inflight request), dispatches the + * pending sync acks immediately. + * + * @param rk Client instance. + * @param rkcg Consumer group handle. + * + * @locality main thread + */ +static void rd_kafka_share_dispatch_pending_sync_acks(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg) { + rd_kafka_broker_t *rkb; + + rd_kafka_rdlock(rk); + TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) { + if (!rkb->rkb_pending_commit_sync.sync_ack_details) + continue; + + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt++; + + if (rkb->rkb_share_fetch_enqueued || + rd_kafka_broker_or_instance_terminating(rkb)) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Broker %s has in-flight request, " + "sync acks stored in pending_commit_sync", + rd_kafka_broker_name(rkb)); + continue; + } + + /* Broker is free — dispatch pending sync acks */ + rd_kafka_share_enqueue_sync_ack_op(rk, rkb); + } + rd_kafka_rdunlock(rk); +} + +/** + * @brief Main thread handler for SHARE_COMMIT_SYNC_FANOUT op. + * + * Stores the commit_sync request state, segregates acks by leader + * into pending_commit_sync, then dispatches to free brokers and + * starts the timeout timer. + * + * @locality main thread + */ +rd_kafka_op_res_t rd_kafka_share_commit_sync_fanout_op(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_kafka_op_t *rko) { + rd_kafka_cgrp_t *rkcg = rd_kafka_cgrp_get(rk); + rd_list_t *ack_batches = + rko->rko_u.share_commit_sync_fanout.ack_batches; + rd_ts_t abs_timeout = rko->rko_u.share_commit_sync_fanout.abs_timeout; + rd_kafka_topic_partition_list_t *results = + rko->rko_u.share_commit_sync_fanout.results; + int i; + int64_t timeout_us; + + /* Take ownership from op */ + rko->rko_u.share_commit_sync_fanout.ack_batches = NULL; + rko->rko_u.share_commit_sync_fanout.results = NULL; + + /* Initialize commit_sync request state */ + rkcg->rkcg_share.commit_sync_request_id_counter++; + if (rkcg->rkcg_share.commit_sync_request_id_counter == INT64_MAX) + rkcg->rkcg_share.commit_sync_request_id_counter = 1; + rkcg->rkcg_commit_sync_request.id = + rkcg->rkcg_share.commit_sync_request_id_counter; + rkcg->rkcg_commit_sync_request.abs_timeout = abs_timeout; + rkcg->rkcg_commit_sync_request.replyq = rko->rko_replyq.q; + rkcg->rkcg_commit_sync_request.results = results; + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt = 0; + + rko->rko_replyq.q = NULL; /* Ownership transferred to rkcg */ + + /* Initialize all partition errors to IN_PROGRESS sentinel */ + for (i = 0; i < results->cnt; i++) + results->elems[i].err = RD_KAFKA_RESP_ERR__IN_PROGRESS; + + if (!ack_batches || rd_list_cnt(ack_batches) == 0) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "No acks to commit in " + "SHARE_COMMIT_SYNC_FANOUT, " + "sending immediate response"); + + for (i = 0; i < results->cnt; i++) + results->elems[i].err = RD_KAFKA_RESP_ERR_NO_ERROR; + + rd_kafka_share_commit_sync_send_response(rkcg); + RD_IF_FREE(ack_batches, rd_list_destroy); + return RD_KAFKA_OP_RES_HANDLED; + } + + /* Phase 1: Segregate all acks into rkb_pending_commit_sync */ + rd_kafka_share_segregate_sync_acks_by_leader(rk, rkcg, ack_batches, + abs_timeout); + + /* Phase 2: Dispatch pending sync acks to free brokers */ + rd_kafka_share_dispatch_pending_sync_acks(rk, rkcg); + + if (rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt == 0) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "No brokers available for commit sync, " + "sending immediate response"); + rd_kafka_share_commit_sync_send_response(rkcg); + return RD_KAFKA_OP_RES_HANDLED; + } + + /* Start timeout timer */ + timeout_us = abs_timeout - rd_clock(); + if (timeout_us <= 0) + timeout_us = 1; + + rd_kafka_timer_start_oneshot( + &rk->rk_timers, &rkcg->rkcg_commit_sync_request.tmr, + rd_true /*restart*/, timeout_us, + rd_kafka_share_commit_sync_timeout_cb, NULL); + + rd_kafka_dbg(rk, CGRP, "SHARE", + "Commit sync fanout: waiting for %d broker results, " + "timeout in %" PRId64 " ms", + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt, + timeout_us / 1000); + + return RD_KAFKA_OP_RES_HANDLED; +} + +/** + * @brief Asynchronously commit all pending acknowledgements. + * + * Builds ack details from the inflight map and enqueues a + * SHARE_COMMIT_ASYNC_FANOUT op on the main thread to send + * them to brokers. Does not fetch new records. + * + * In implicit ack mode, all ACQUIRED records are first converted + * to ACCEPT before building the ack details. + * + * @param rkshare Share consumer instance. + * @returns NULL on success, or an rd_kafka_error_t* on failure. + * + * @locality app thread + */ +rd_kafka_error_t *rd_kafka_share_commit_async(rd_kafka_share_t *rkshare) { + rd_kafka_t *rk = rkshare->rkshare_rk; + rd_kafka_op_t *rko; + rd_list_t *ack_batches; + rd_kafka_error_t *error = NULL; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + if (unlikely((error = rd_kafka_share_consumer_closed_error(rkshare)) != + NULL)) + goto done; + + rd_kafka_dbg(rk, CGRP, "SHARE", "Committing asynchronously"); + + /* Drain rk_rep for all pending callbacks (non-blocking) */ + rd_kafka_q_serve(rk->rk_rep, RD_POLL_NOWAIT, 0, RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); + + rd_kafka_share_acknowledge_all_if_implicit(rkshare); + + ack_batches = rd_kafka_share_build_ack_details(rk->rk_rkshare); + + if (!ack_batches) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "No pending acknowledgements to commit"); + goto done; + } + + rko = rd_kafka_op_new_cb(rk, RD_KAFKA_OP_SHARE_COMMIT_ASYNC_FANOUT, + rd_kafka_share_commit_async_fanout_op); + rko->rko_u.share_commit_async_fanout.ack_batches = ack_batches; + + rd_kafka_q_enq(rk->rk_ops, rko); + +done: + rd_kafka_share_release(rkshare); + return error; +} + +/** + * @brief Synchronously commit all pending acknowledgements. + * + * Builds ack details from the inflight map and enqueues a + * SHARE_COMMIT_SYNC_FANOUT op on the main thread. Blocks the + * app thread on a temp queue until the main thread sends a + * response (or timeout fires). + * + * In implicit ack mode, all ACQUIRED records are first converted + * to ACCEPT before building the ack details. + * + * @param rkshare Share consumer instance. + * @param timeout_ms Timeout in milliseconds. + * @param partitions [out] Per-partition results. Each partition's err + * field contains the result. The caller must destroy + * the returned list with + * rd_kafka_topic_partition_list_destroy(). + * Set to NULL if no acks were pending. + * + * @returns NULL on success, or an rd_kafka_error_t* on failure. + * The returned error must be freed with rd_kafka_error_destroy(). + * + * @locality app thread + */ +rd_kafka_error_t * +rd_kafka_share_commit_sync(rd_kafka_share_t *rkshare, + int timeout_ms, + rd_kafka_topic_partition_list_t **partitions) { + rd_kafka_t *rk = rkshare->rkshare_rk; + rd_kafka_op_t *rko; + rd_kafka_op_t *rko_reply; + rd_list_t *ack_batches; + rd_kafka_q_t *tmpq; + rd_ts_t abs_timeout; + rd_kafka_topic_partition_list_t *results; + rd_kafka_error_t *error = NULL; + int i; + + *partitions = NULL; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + if (unlikely((error = rd_kafka_share_consumer_closed_error(rkshare)) != + NULL)) + goto done; + + rd_kafka_dbg(rk, CGRP, "SHARE", + "Committing synchronously with timeout %d ms", timeout_ms); + + /* Drain rk_rep for all pending callbacks (non-blocking) */ + rd_kafka_q_serve(rk->rk_rep, RD_POLL_NOWAIT, 0, RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); + + rd_kafka_share_acknowledge_all_if_implicit(rkshare); + + ack_batches = rd_kafka_share_build_ack_details(rk->rk_rkshare); + + if (!ack_batches) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "No pending acknowledgements to commit"); + goto done; + } + + abs_timeout = rd_timeout_init(timeout_ms); + + /* Build per-partition results list from ack batches */ + results = rd_kafka_topic_partition_list_new(rd_list_cnt(ack_batches)); + for (i = 0; i < rd_list_cnt(ack_batches); i++) { + rd_kafka_share_ack_batches_t *batch = + rd_list_elem(ack_batches, i); + rd_kafka_topic_partition_list_add_copy(results, batch->rktpar); + } + + /* Create temp queue for blocking */ + tmpq = rd_kafka_q_new(rk); + + /* Create SYNC_FANOUT op */ + rko = rd_kafka_op_new_cb(rk, RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT, + rd_kafka_share_commit_sync_fanout_op); + rko->rko_u.share_commit_sync_fanout.ack_batches = ack_batches; + rko->rko_u.share_commit_sync_fanout.abs_timeout = abs_timeout; + rko->rko_u.share_commit_sync_fanout.results = results; + rko->rko_replyq = RD_KAFKA_REPLYQ(tmpq, 0); + + rd_kafka_q_enq(rk->rk_ops, rko); + + /* Block on temp queue — main thread always sends a response + * (either from broker replies or timeout callback) */ + rko_reply = rd_kafka_q_pop(tmpq, RD_POLL_INFINITE, 0); + + rd_kafka_q_destroy_owner(tmpq); + + /* Extract per-partition results from reply */ + rd_dassert(rko_reply->rko_type == + RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT_REPLY); + *partitions = rko_reply->rko_u.share_commit_sync_fanout_reply.results; + rko_reply->rko_u.share_commit_sync_fanout_reply.results = NULL; + + rd_kafka_op_destroy(rko_reply); + + /* Drain rk_rep for callbacks again before returning */ + rd_kafka_q_serve(rk->rk_rep, RD_POLL_NOWAIT, 0, RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); + +done: + rd_kafka_share_release(rkshare); + return error; +} + +/** + * Schedules a rebootstrap of the cluster immediately. + * + * @locks none + * @locks_acquired rd_kafka_timers_lock() + * @locality any + */ +void rd_kafka_rebootstrap(rd_kafka_t *rk) { + if (rk->rk_conf.metadata_recovery_strategy == + RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE) + return; + + if (rd_atomic32_set(&rk->rk_rebootstrap_in_progress, 1) == 0) { + /* Only when not already in progress 0 -> 1. + * After setting down a learned broker it could reconnect and + * disconnect again before previous reboostrap completes, + * causing a new re-bootstrap. */ + rd_kafka_timer_start_oneshot( + &rk->rk_timers, &rk->rebootstrap_tmr, rd_true /*restart*/, + 0, rd_kafka_rebootstrap_tmr_cb, NULL); + } +} + +/** + * Starts rebootstrap timer with the configured interval. Only if not + * started or stopped. + * + * @locks none + * @locks_acquired rd_kafka_timers_lock() + * @locality any + */ +void rd_kafka_rebootstrap_tmr_start_maybe(rd_kafka_t *rk) { + if (rk->rk_conf.metadata_recovery_strategy == + RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE) + return; + + rd_kafka_timer_start_oneshot( + &rk->rk_timers, &rk->rebootstrap_tmr, rd_false /*don't restart*/, + rk->rk_conf.metadata_recovery_rebootstrap_trigger_ms * 1000LL, + rd_kafka_rebootstrap_tmr_cb, NULL); +} + +/** + * Stops rebootstrap timer, for example after a successful metadata response. + * + * @return 1 if the timer was started (before being stopped), else 0. + * + * @locks none + * @locks_acquired rd_kafka_timers_lock() + * @locality any + */ +int rd_kafka_rebootstrap_tmr_stop(rd_kafka_t *rk) { + if (rk->rk_conf.metadata_recovery_strategy == + RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE) + return 0; + + return rd_kafka_timer_stop(&rk->rk_timers, &rk->rebootstrap_tmr, + rd_true /* lock */); +} + +/** + * Counts usage of the legacy/simple consumer (rd_kafka_consume_start() with + * friends) since it does not have an API for stopping the cgrp we will need to + * sort that out automatically in the background when all consumption + * has stopped. + * + * Returns 0 if a High level consumer is already instantiated + * which means a Simple consumer cannot co-operate with it, else 1. + * + * A rd_kafka_t handle can never migrate from simple to high-level, or + * vice versa, so we dont need a ..consumer_del(). + */ +int rd_kafka_simple_consumer_add(rd_kafka_t *rk) { + if (rd_atomic32_get(&rk->rk_simple_cnt) < 0) + return 0; + + return (int)rd_atomic32_add(&rk->rk_simple_cnt, 1); +} + + + +/** + * rktp fetch is split up in these parts: + * * application side: + * * broker side (handled by current leader broker thread for rktp): + * - the fetch state, initial offset, etc. + * - fetching messages, updating fetched offset, etc. + * - offset commits + * + * Communication between the two are: + * app side -> rdkafka main side: rktp_ops + * broker thread -> app side: rktp_fetchq + * + * There is no shared state between these threads, instead + * state is communicated through the two op queues, and state synchronization + * is performed by version barriers. + * + */ + +static RD_UNUSED int rd_kafka_consume_start0(rd_kafka_topic_t *rkt, + int32_t partition, + int64_t offset, + rd_kafka_q_t *rkq) { + rd_kafka_toppar_t *rktp; + + if (partition < 0) { + rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION, + ESRCH); + return -1; + } + + if (!rd_kafka_simple_consumer_add(rkt->rkt_rk)) { + rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, EINVAL); + return -1; + } + + rd_kafka_topic_wrlock(rkt); + rktp = rd_kafka_toppar_desired_add(rkt, partition); + rd_kafka_topic_wrunlock(rkt); + + /* Verify offset */ + if (offset == RD_KAFKA_OFFSET_BEGINNING || + offset == RD_KAFKA_OFFSET_END || + offset <= RD_KAFKA_OFFSET_TAIL_BASE) { + /* logical offsets */ + + } else if (offset == RD_KAFKA_OFFSET_STORED) { + /* offset manager */ + + if (rkt->rkt_conf.offset_store_method == + RD_KAFKA_OFFSET_METHOD_BROKER && + RD_KAFKAP_STR_IS_NULL(rkt->rkt_rk->rk_group_id)) { + /* Broker based offsets require a group id. */ + rd_kafka_toppar_destroy(rktp); + rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, + EINVAL); + return -1; + } + + } else if (offset < 0) { + rd_kafka_toppar_destroy(rktp); + rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, EINVAL); + return -1; + } + + rd_kafka_toppar_op_fetch_start(rktp, RD_KAFKA_FETCH_POS(offset, -1), + rkq, RD_KAFKA_NO_REPLYQ); + + rd_kafka_toppar_destroy(rktp); + + rd_kafka_set_last_error(0, 0); + return 0; +} + + + +int rd_kafka_consume_start(rd_kafka_topic_t *app_rkt, + int32_t partition, + int64_t offset) { + rd_kafka_topic_t *rkt = rd_kafka_topic_proper(app_rkt); + rd_kafka_dbg(rkt->rkt_rk, TOPIC, "START", + "Start consuming partition %" PRId32, partition); + return rd_kafka_consume_start0(rkt, partition, offset, NULL); +} + +int rd_kafka_consume_start_queue(rd_kafka_topic_t *app_rkt, + int32_t partition, + int64_t offset, + rd_kafka_queue_t *rkqu) { + rd_kafka_topic_t *rkt = rd_kafka_topic_proper(app_rkt); + + return rd_kafka_consume_start0(rkt, partition, offset, rkqu->rkqu_q); +} + + + +static RD_UNUSED int rd_kafka_consume_stop0(rd_kafka_toppar_t *rktp) { + rd_kafka_q_t *tmpq = NULL; + rd_kafka_resp_err_t err; + + rd_kafka_topic_wrlock(rktp->rktp_rkt); + rd_kafka_toppar_lock(rktp); + rd_kafka_toppar_desired_del(rktp); + rd_kafka_toppar_unlock(rktp); + rd_kafka_topic_wrunlock(rktp->rktp_rkt); + + tmpq = rd_kafka_q_new(rktp->rktp_rkt->rkt_rk); + + rd_kafka_toppar_op_fetch_stop(rktp, RD_KAFKA_REPLYQ(tmpq, 0)); + + /* Synchronisation: Wait for stop reply from broker thread */ + err = rd_kafka_q_wait_result(tmpq, RD_POLL_INFINITE); + rd_kafka_q_destroy_owner(tmpq); + + rd_kafka_set_last_error(err, err ? EINVAL : 0); return err ? -1 : 0; } @@ -3218,12 +4965,12 @@ static rd_kafka_op_res_t rd_kafka_consume_callback0( struct consume_ctx ctx = {.consume_cb = consume_cb, .opaque = opaque}; rd_kafka_op_res_t res; - rd_kafka_app_poll_start(rkq->rkq_rk, 0, timeout_ms); + rd_kafka_app_poll_start(rkq->rkq_rk, rkq, 0, timeout_ms); res = rd_kafka_q_serve(rkq, timeout_ms, max_cnt, RD_KAFKA_Q_CB_RETURN, rd_kafka_consume_cb, &ctx); - rd_kafka_app_polled(rkq->rkq_rk); + rd_kafka_app_polled(rkq->rkq_rk, rkq); return res; } @@ -3289,7 +5036,7 @@ rd_kafka_consume0(rd_kafka_t *rk, rd_kafka_q_t *rkq, int timeout_ms) { rd_ts_t now = rd_clock(); rd_ts_t abs_timeout = rd_timeout_init0(now, timeout_ms); - rd_kafka_app_poll_start(rk, now, timeout_ms); + rd_kafka_app_poll_start(rk, rkq, now, timeout_ms); rd_kafka_yield_thread = 0; while (( @@ -3306,7 +5053,7 @@ rd_kafka_consume0(rd_kafka_t *rk, rd_kafka_q_t *rkq, int timeout_ms) { /* Callback called rd_kafka_yield(), we must * stop dispatching the queue and return. */ rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INTR, EINTR); - rd_kafka_app_polled(rk); + rd_kafka_app_polled(rk, rkq); return NULL; } @@ -3318,7 +5065,7 @@ rd_kafka_consume0(rd_kafka_t *rk, rd_kafka_q_t *rkq, int timeout_ms) { /* Timeout reached with no op returned. */ rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__TIMED_OUT, ETIMEDOUT); - rd_kafka_app_polled(rk); + rd_kafka_app_polled(rk, rkq); return NULL; } @@ -3333,7 +5080,7 @@ rd_kafka_consume0(rd_kafka_t *rk, rd_kafka_q_t *rkq, int timeout_ms) { rd_kafka_set_last_error(0, 0); - rd_kafka_app_polled(rk); + rd_kafka_app_polled(rk, rkq); return rkmessage; } @@ -3384,7 +5131,6 @@ rd_kafka_resp_err_t rd_kafka_poll_set_consumer(rd_kafka_t *rk) { } - rd_kafka_message_t *rd_kafka_consumer_poll(rd_kafka_t *rk, int timeout_ms) { rd_kafka_cgrp_t *rkcg; @@ -3424,17 +5170,34 @@ static rd_kafka_error_t *rd_kafka_consumer_close_q(rd_kafka_t *rk, /* If a fatal error has been raised and this is an * explicit consumer_close() from the application we return * a fatal error. Otherwise let the "silent" no_consumer_close - * logic be performed to clean up properly. */ - if (!rd_kafka_destroy_flags_no_consumer_close(rk) && + * logic be performed to clean up properly. + * + * Share consumers are exempt: an explicit close() must still + * flush pending acks and send the share-session leave even when + * a fatal error is set. The destroy-after-fatal path is + * unaffected because it sets NO_CONSUMER_CLOSE (so the + * !no_consumer_close clause is already false there, and the cgrp + * terminate handler drops the acks regardless). */ + if (!RD_KAFKA_IS_SHARE_CONSUMER(rk) && + !rd_kafka_destroy_flags_no_consumer_close(rk) && (error = rd_kafka_get_fatal_error(rk))) return error; rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "CLOSE", "Closing consumer"); - /* Redirect cgrp queue to the rebalance queue to make sure all posted - * ops (e.g., rebalance callbacks) are served by - * the application/caller. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + /* Unlike regular consumer, the rk_rep queue is not forwarded to + * rkcg_q by default, so we need this additional step to ensure + * callbacks from rk_rep queue can be served by the + * application/caller */ + rd_kafka_q_fwd_set(rk->rk_rep, rkq); + } + + /* For regular consumers, this redirects cgrp queue to the rebalance + * queue to make sure all posted ops (e.g., rebalance callbacks) are + * served by the application/caller. For share consumers, this will take + * care of forwarding only the cgrp ops */ rd_kafka_q_fwd_set(rkcg->rkcg_q, rkq); /* Tell cgrp subsystem to terminate. A TERMINATE op will be posted @@ -3452,10 +5215,42 @@ rd_kafka_error_t *rd_kafka_consumer_close_queue(rd_kafka_t *rk, return rd_kafka_consumer_close_q(rk, rkqu->rkqu_q); } -rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t *rk) { +/** + * @brief Asynchronously close the share consumer. + * @param rkshare Share consumer instance. + * @param rkqu Queue for forwarding events/callbacks during close. + * @returns `rd_kafka_error_t *` - NULL on success, error object on failure + */ +rd_kafka_error_t *rd_kafka_share_consumer_close_queue(rd_kafka_share_t *rkshare, + rd_kafka_queue_t *rkqu) { + rd_kafka_error_t *error = NULL; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + /* TODO KIP-932: Guard this with checks for rkshare + * and rkshare->rkshare_rk */ + if (unlikely((error = rd_kafka_share_consumer_closed_error(rkshare)) != + NULL)) + goto done; + + rkshare->rkshare_consumer_closing = rd_true; + error = rd_kafka_consumer_close_queue(rkshare->rkshare_rk, rkqu); + if (error) { + rkshare->rkshare_consumer_closing = rd_false; + goto done; + } + +done: + rd_kafka_share_release(rkshare); + return error; +} + + +rd_kafka_error_t *rd_kafka_consumer_close0(rd_kafka_t *rk) { rd_kafka_error_t *error; - rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR__TIMED_OUT; rd_kafka_q_t *rkq; + rd_bool_t got_terminate = rd_false; /* Create a temporary reply queue to handle the TERMINATE reply op. */ rkq = rd_kafka_q_new(rk); @@ -3463,12 +5258,8 @@ rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t *rk) { /* Initiate the close (async) */ error = rd_kafka_consumer_close_q(rk, rkq); if (error) { - err = rd_kafka_error_is_fatal(error) - ? RD_KAFKA_RESP_ERR__FATAL - : rd_kafka_error_code(error); - rd_kafka_error_destroy(error); rd_kafka_q_destroy_owner(rkq); - return err; + return error; } /* Disable the queue if termination is immediate or the user @@ -3480,7 +5271,7 @@ rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t *rk) { rd_kafka_dbg(rk, CONSUMER, "CLOSE", "Disabling and purging temporary queue to quench " "close events"); - err = RD_KAFKA_RESP_ERR_NO_ERROR; + got_terminate = rd_true; rd_kafka_q_disable(rkq); /* Purge ops already enqueued */ rd_kafka_q_purge(rkq); @@ -3491,7 +5282,9 @@ rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t *rk) { rd_kafka_op_res_t res; if ((rko->rko_type & ~RD_KAFKA_OP_FLAGMASK) == RD_KAFKA_OP_TERMINATE) { - err = rko->rko_err; + error = rko->rko_error; + rko->rko_error = NULL; + got_terminate = rd_true; rd_kafka_op_destroy(rko); break; } @@ -3506,17 +5299,61 @@ rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t *rk) { rd_kafka_q_destroy_owner(rkq); - if (err) + if (!got_terminate) + error = rd_kafka_error_new( + RD_KAFKA_RESP_ERR__TIMED_OUT, "%s", + "Timed out waiting for a TERMINATE reply to arrive"); + + if (error) rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "CLOSE", "Consumer closed with error: %s", - rd_kafka_err2str(err)); + rd_kafka_error_string(error)); else rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "CLOSE", "Consumer closed"); + return error; +} + +rd_kafka_resp_err_t rd_kafka_consumer_close(rd_kafka_t *rk) { + rd_kafka_error_t *error; + rd_kafka_resp_err_t err; + + error = rd_kafka_consumer_close0(rk); + if (!error) + return RD_KAFKA_RESP_ERR_NO_ERROR; + + err = rd_kafka_error_is_fatal(error) ? RD_KAFKA_RESP_ERR__FATAL + : rd_kafka_error_code(error); + rd_kafka_error_destroy(error); return err; } +rd_kafka_error_t *rd_kafka_share_consumer_close(rd_kafka_share_t *rkshare) { + rd_kafka_error_t *error = NULL; + rd_kafka_t *rk; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) + return error; + + /* TODO KIP-932: Guard this with checks for rkshare and + * rkshare->rkshare_rk. Check if this is needed for other APIs + * as well. + */ + if (unlikely((error = rd_kafka_share_consumer_closed_error(rkshare)) != + NULL)) + goto done; + + rk = rkshare->rkshare_rk; + rkshare->rkshare_consumer_closing = rd_true; + error = rd_kafka_consumer_close0(rk); + rkshare->rkshare_consumer_closing = rd_false; + +done: + rd_kafka_share_release(rkshare); + return error; +} + int rd_kafka_consumer_closed(rd_kafka_t *rk) { if (unlikely(!rk->rk_cgrp)) @@ -3525,6 +5362,15 @@ int rd_kafka_consumer_closed(rd_kafka_t *rk) { return rd_atomic32_get(&rk->rk_cgrp->rkcg_terminated); } +/** + * @brief Check if the share consumer is closed. + * @param rkshare Share consumer instance. + * @returns 1 if the consumer is closed, else 0. + */ +int rd_kafka_share_consumer_closed(rd_kafka_share_t *rkshare) { + return rd_kafka_consumer_closed(rkshare->rkshare_rk); +} + rd_kafka_resp_err_t rd_kafka_committed(rd_kafka_t *rk, @@ -3721,6 +5567,7 @@ rd_kafka_resp_err_t rd_kafka_query_watermark_offsets(rd_kafka_t *rk, struct rd_kafka_partition_leader *leader; rd_list_t leaders; rd_kafka_resp_err_t err; + int tmout; partitions = rd_kafka_topic_partition_list_new(1); rktpar = @@ -3767,9 +5614,13 @@ rd_kafka_resp_err_t rd_kafka_query_watermark_offsets(rd_kafka_t *rk, /* Wait for reply (or timeout) */ while (state.err == RD_KAFKA_RESP_ERR__IN_PROGRESS) { - rd_kafka_q_serve(rkq, RD_POLL_INFINITE, 0, - RD_KAFKA_Q_CB_CALLBACK, rd_kafka_poll_cb, - NULL); + tmout = rd_timeout_remains(ts_end); + if (rd_timeout_expired(tmout)) { + state.err = RD_KAFKA_RESP_ERR__TIMED_OUT; + break; + } + rd_kafka_q_serve(rkq, tmout, 0, RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); } rd_kafka_q_destroy_owner(rkq); @@ -3973,6 +5824,10 @@ rd_kafka_op_res_t rd_kafka_poll_cb(rd_kafka_t *rk, return RD_KAFKA_OP_RES_PASS; /* Return as event */ } + // rd_kafka_dbg(rk, CGRP, "CB", "Received cb_type %d op_type 0x%04x", + // cb_type, + // rko->rko_type); + switch ((int)rko->rko_type) { case RD_KAFKA_OP_FETCH: if (!rk->rk_conf.consume_cb || @@ -3980,10 +5835,10 @@ rd_kafka_op_res_t rd_kafka_poll_cb(rd_kafka_t *rk, cb_type == RD_KAFKA_Q_CB_FORCE_RETURN) return RD_KAFKA_OP_RES_PASS; /* Dont handle here */ else { - rk->rk_ts_last_poll_end = rd_clock(); - struct consume_ctx ctx = {.consume_cb = - rk->rk_conf.consume_cb, - .opaque = rk->rk_conf.opaque}; + rkq->rkq_ts_last_poll_end = rd_clock(); + struct consume_ctx ctx = {.consume_cb = + rk->rk_conf.consume_cb, + .opaque = rk->rk_conf.opaque}; return rd_kafka_consume_cb(rk, rkq, rko, cb_type, &ctx); } @@ -4016,6 +5871,24 @@ rd_kafka_op_res_t rd_kafka_poll_cb(rd_kafka_t *rk, rko->rko_u.offset_commit.opaque); break; + case RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE: { + rd_kafka_share_t *rkshare = rk->rk_rkshare; + /* Lookup callback at invoke time. + * Locality: app thread */ + if (!rkshare || + !rk->rk_share_consumer.share_acknowledgement_commit_cb) + return RD_KAFKA_OP_RES_PASS; /* Dont handle here */ + /* Set reentrancy flag so share consumer APIs called from + * within the callback can detect and reject the call. */ + rk->rk_share_consumer.in_callback = rd_true; + rk->rk_share_consumer.share_acknowledgement_commit_cb( + rkshare, rko->rko_u.share_ack_commit_cb_execute.partitions, + rko->rko_err, + rk->rk_share_consumer.acknowledgement_commit_cb_opaque); + rk->rk_share_consumer.in_callback = rd_false; + break; + } + case RD_KAFKA_OP_FETCH_STOP | RD_KAFKA_OP_REPLY: /* Reply from toppar FETCH_STOP */ rd_kafka_assignment_partition_stopped(rk, rko->rko_rktp); @@ -4191,6 +6064,18 @@ rd_kafka_op_res_t rd_kafka_poll_cb(rd_kafka_t *rk, res = rd_kafka_metadata_update_op(rk, rko->rko_u.metadata.mdi); break; + case RD_KAFKA_OP_SHARE_FETCH_RESPONSE: + /* TODO KIP-932: RD_KAFKA_OP_SHARE_FETCH_RESPONSE should never + be handled from this cb in Share Consumer except maybe in + case of closing as this op we don't forward rk_rep queue to + rkcg.q. This op is handled separately in + rd_kafka_share_poll() during normal working of Share + Consumer. Check what is the correct way to handle this op + while closing. */ + rd_kafka_assert(rk, rk->rk_rkshare->rkshare_consumer_closing); + res = RD_KAFKA_OP_RES_PASS; + break; + default: /* If op has a callback set (e.g., OAUTHBEARER_REFRESH), * call it. */ @@ -4213,8 +6098,9 @@ rd_kafka_op_res_t rd_kafka_poll_cb(rd_kafka_t *rk, int rd_kafka_poll(rd_kafka_t *rk, int timeout_ms) { int r; - r = rd_kafka_q_serve(rk->rk_rep, timeout_ms, 0, RD_KAFKA_Q_CB_CALLBACK, - rd_kafka_poll_cb, NULL); + r = rd_kafka_q_serve_maybe_consume(rk->rk_rep, timeout_ms, 0, + RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); return r; } @@ -4222,8 +6108,9 @@ int rd_kafka_poll(rd_kafka_t *rk, int timeout_ms) { rd_kafka_event_t *rd_kafka_queue_poll(rd_kafka_queue_t *rkqu, int timeout_ms) { rd_kafka_op_t *rko; - rko = rd_kafka_q_pop_serve(rkqu->rkqu_q, rd_timeout_us(timeout_ms), 0, - RD_KAFKA_Q_CB_EVENT, rd_kafka_poll_cb, NULL); + rko = rd_kafka_q_pop_serve_maybe_consume( + rkqu->rkqu_q, rd_timeout_us(timeout_ms), 0, RD_KAFKA_Q_CB_EVENT, + rd_kafka_poll_cb, NULL); if (!rko) @@ -4235,8 +6122,9 @@ rd_kafka_event_t *rd_kafka_queue_poll(rd_kafka_queue_t *rkqu, int timeout_ms) { int rd_kafka_queue_poll_callback(rd_kafka_queue_t *rkqu, int timeout_ms) { int r; - r = rd_kafka_q_serve(rkqu->rkqu_q, timeout_ms, 0, - RD_KAFKA_Q_CB_CALLBACK, rd_kafka_poll_cb, NULL); + r = rd_kafka_q_serve_maybe_consume(rkqu->rkqu_q, timeout_ms, 0, + RD_KAFKA_Q_CB_CALLBACK, + rd_kafka_poll_cb, NULL); return r; } @@ -5181,13 +7069,8 @@ const char *rd_kafka_get_debug_contexts(void) { int rd_kafka_path_is_dir(const char *path) { -#ifdef _WIN32 - struct _stat st; - return (_stat(path, &st) == 0 && st.st_mode & S_IFDIR); -#else - struct stat st; - return (stat(path, &st) == 0 && S_ISDIR(st.st_mode)); -#endif + rd_bool_t is_dir; + return rd_file_stat(path, &is_dir) && is_dir; } @@ -5296,18 +7179,12 @@ rd_kafka_Uuid_t *rd_kafka_Uuid_copy(const rd_kafka_Uuid_t *uuid) { * @remark Must be freed after use using rd_kafka_Uuid_destroy(). */ rd_kafka_Uuid_t rd_kafka_Uuid_random() { - int i; unsigned char rand_values_bytes[16] = {0}; uint64_t *rand_values_uint64 = (uint64_t *)rand_values_bytes; - unsigned char *rand_values_app; - rd_kafka_Uuid_t ret = RD_KAFKA_UUID_ZERO; - for (i = 0; i < 16; i += 2) { - uint16_t rand_uint16 = (uint16_t)rd_jitter(0, INT16_MAX - 1); - /* No need to convert endianess here because it's still only - * a random value. */ - rand_values_app = (unsigned char *)&rand_uint16; - rand_values_bytes[i] |= rand_values_app[0]; - rand_values_bytes[i + 1] |= rand_values_app[1]; + rd_kafka_Uuid_t ret = RD_KAFKA_UUID_ZERO; + if (!rd_rand_bytes(rand_values_bytes, sizeof(rand_values_bytes))) { + rd_assert(!*"BUG: a random UUID cannot be requested without " + "a reliable entropy source"); } rand_values_bytes[6] &= 0x0f; /* clear version */ @@ -5339,7 +7216,7 @@ void rd_kafka_Uuid_destroy(rd_kafka_Uuid_t *uuid) { * * @remark Must be freed after use. */ -const char *rd_kafka_Uuid_str(const rd_kafka_Uuid_t *uuid) { +char *rd_kafka_Uuid_str(const rd_kafka_Uuid_t *uuid) { int i, j; unsigned char bytes[16]; char *ret = rd_calloc(37, sizeof(*ret)); diff --git a/lib/librdkafka-2.10.1/src/rdkafka.h b/lib/librdkafka-2.15.0/src/rdkafka.h similarity index 88% rename from lib/librdkafka-2.10.1/src/rdkafka.h rename to lib/librdkafka-2.15.0/src/rdkafka.h index d5a679cf138..5eb511fcb83 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka.h +++ b/lib/librdkafka-2.15.0/src/rdkafka.h @@ -167,7 +167,7 @@ typedef SSIZE_T ssize_t; * @remark This value should only be used during compile time, * for runtime checks of version use rd_kafka_version() */ -#define RD_KAFKA_VERSION 0x020a01ff +#define RD_KAFKA_VERSION 0x020f00ff /** * @brief Returns the librdkafka version as integer. @@ -265,6 +265,7 @@ typedef struct rd_kafka_acl_result_s rd_kafka_acl_result_t; typedef struct rd_kafka_Uuid_s rd_kafka_Uuid_t; typedef struct rd_kafka_topic_partition_result_s rd_kafka_topic_partition_result_t; +typedef struct rd_kafka_share_s rd_kafka_share_t; /* @endcond */ @@ -640,6 +641,8 @@ typedef enum { RD_KAFKA_RESP_ERR_PRINCIPAL_DESERIALIZATION_FAILURE = 97, /** Unknown Topic Id */ RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID = 100, + /** The log's topic ID did not match the topic ID in the request */ + RD_KAFKA_RESP_ERR_INCONSISTENT_TOPIC_ID = 103, /** The member epoch is fenced by the group coordinator */ RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH = 110, /** The instance ID is still used by another member in the @@ -650,15 +653,54 @@ typedef enum { RD_KAFKA_RESP_ERR_UNSUPPORTED_ASSIGNOR = 112, /** The member epoch is stale */ RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH = 113, + /** The request was sent to an endpoint of the wrong type. */ + RD_KAFKA_RESP_ERR_MISMATCHED_ENDPOINT_TYPE = 114, + /** This endpoint type is not supported yet. */ + RD_KAFKA_RESP_ERR_UNSUPPORTED_ENDPOINT_TYPE = 115, + /** This controller ID is not known. */ + RD_KAFKA_RESP_ERR_UNKNOWN_CONTROLLER_ID = 116, /** Client sent a push telemetry request with an invalid or outdated * subscription ID. */ RD_KAFKA_RESP_ERR_UNKNOWN_SUBSCRIPTION_ID = 117, /** Client sent a push telemetry request larger than the maximum size * the broker will accept. */ RD_KAFKA_RESP_ERR_TELEMETRY_TOO_LARGE = 118, + /** The controller has considered the broker registration + * to be invalid. */ + RD_KAFKA_RESP_ERR_INVALID_REGISTRATION = 119, + /** The server encountered an error with the transaction. + * The client can abort the transaction to continue using + * this transactional ID. */ + RD_KAFKA_RESP_ERR_TRANSACTION_ABORTABLE = 120, + /** The record state is invalid. The acknowledgement of delivery + * could not be completed. */ + RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE = 121, + /** The share session was not found. */ + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND = 122, + /** The share session epoch is invalid. */ + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH = 123, + /** The share coordinator rejected the request because the + * share-group state epoch did not match. */ + RD_KAFKA_RESP_ERR_FENCED_STATE_EPOCH = 124, + /** The voter key doesn't match the receiving replica's key. */ + RD_KAFKA_RESP_ERR_INVALID_VOTER_KEY = 125, + /** The voter is already part of the set of voters. */ + RD_KAFKA_RESP_ERR_DUPLICATE_VOTER = 126, + /** The voter is not part of the set of voters. */ + RD_KAFKA_RESP_ERR_VOTER_NOT_FOUND = 127, + /** The regular expression is not valid. */ + RD_KAFKA_RESP_ERR_INVALID_REGULAR_EXPRESSION = 128, /** Client metadata is stale, * client should rebootstrap to obtain new metadata. */ RD_KAFKA_RESP_ERR_REBOOTSTRAP_REQUIRED = 129, + /** The supplied topology is invalid. */ + RD_KAFKA_RESP_ERR_STREAMS_INVALID_TOPOLOGY = 130, + /** The supplied topology epoch is invalid. */ + RD_KAFKA_RESP_ERR_STREAMS_INVALID_TOPOLOGY_EPOCH = 131, + /** The supplied topology epoch is outdated. */ + RD_KAFKA_RESP_ERR_STREAMS_TOPOLOGY_FENCED = 132, + /** The limit of share sessions has been reached. */ + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED = 133, RD_KAFKA_RESP_ERR_END_ALL, } rd_kafka_resp_err_t; @@ -1496,6 +1538,61 @@ typedef struct rd_kafka_message_s { } rd_kafka_message_t; +/** + * @brief Opaque container of messages returned by rd_kafka_share_poll(). + * + * Use rd_kafka_messages_count() and rd_kafka_messages_get() to iterate the + * batch, and rd_kafka_messages_destroy() to release it. + */ +typedef struct rd_kafka_messages_s rd_kafka_messages_t; + + +/** + * @brief Returns the number of messages in \p messages. + * + * @param messages Message list returned by rd_kafka_share_poll(). + * May be NULL, in which case 0 is returned. + * + * @returns The number of messages in \p messages. + */ +RD_EXPORT +size_t rd_kafka_messages_count(const rd_kafka_messages_t *messages); + +/** + * @brief Returns the message at zero-based \p index in \p messages. + * + * The returned message is owned by \p messages and remains valid until + * rd_kafka_messages_destroy() is called. + * + * @param messages Message list returned by rd_kafka_share_poll(). + * Must be non-NULL. + * @param index Zero-based index of the message to return. + * Must be strictly less than rd_kafka_messages_count(messages); + * behavior is undefined otherwise. + * + * @returns The message at \p index. + */ +RD_EXPORT +rd_kafka_message_t *rd_kafka_messages_get(const rd_kafka_messages_t *messages, + size_t index); + +/** + * @brief Destroys \p messages and all messages it contains. + * + * @param messages Message list to destroy. NULL is a safe no-op. + * + * @remark After this call the \p messages handle itself, and any + * \c rd_kafka_message_t pointer previously obtained from it via + * rd_kafka_messages_get(), are invalid and must not be accessed — + * doing so reads freed memory. Users should only use this method to + * destroy the \p messages handle returned by rd_kafka_share_poll() and + * set the pointer to NULL after destroying to avoid use-after-free + * bugs. + */ +RD_EXPORT +void rd_kafka_messages_destroy(rd_kafka_messages_t *messages); + + /** * @brief Frees resources for \p rkmessage and hands ownership back to rdkafka. */ @@ -1665,6 +1762,19 @@ rd_kafka_message_status(const rd_kafka_message_t *rkmessage); RD_EXPORT int32_t rd_kafka_message_leader_epoch(const rd_kafka_message_t *rkmessage); +/** + * @brief Get the delivery count for a message associated with the share + * consumer. + * + * @param rkmessage Message object. + * + * @returns Delivery count (number of times this message has been delivered). + * Returns 0 if delivery count is not available like in case of normal + * consumer or producer. + */ +RD_EXPORT int16_t +rd_kafka_message_delivery_count(const rd_kafka_message_t *rkmessage); + /**@}*/ @@ -3082,6 +3192,105 @@ void rd_kafka_destroy(rd_kafka_t *rk); RD_EXPORT void rd_kafka_destroy_flags(rd_kafka_t *rk, int flags); +/** + * @brief Get the SASL callback queue for a share consumer. + * + * @returns a reference to the SASL callback queue for the share consumer, + * if a SASL mechanism with callbacks is configured + * (currently only OAUTHBEARER), else returns NULL. + * + * @remark The dedicated SASL callback queue is only created when + * rd_kafka_conf_enable_sasl_queue() was enabled on the configuration + * before creating the share consumer; otherwise SASL callbacks are + * served on the main queue and this returns NULL. + * + * Use rd_kafka_queue_destroy() to lose the reference. + * + * @sa rd_kafka_conf_enable_sasl_queue() + * @sa rd_kafka_queue_get_sasl() + * @sa rd_kafka_share_sasl_background_callbacks_enable() + */ +RD_EXPORT +rd_kafka_queue_t *rd_kafka_share_queue_get_sasl(rd_kafka_share_t *rkshare); + +/** + * @brief Set SASL/OAUTHBEARER token and metadata for a share consumer. + * + * @param rkshare Share consumer instance. + * @param token_value the mandatory token value to set, often (but not + * necessarily) a JWS compact serialization as per + * https://tools.ietf.org/html/rfc7515#section-3.1. + * @param md_lifetime_ms when the token expires, in terms of the number of + * milliseconds since the epoch. + * @param md_principal_name the mandatory Kafka principal name associated + * with the token. + * @param extensions optional SASL extensions key-value array with + * \p extension_size elements (number of keys * 2), where [i] is the key and + * [i+1] is the key's value, to be communicated to the broker + * as additional key-value pairs during the initial client response as per + * https://tools.ietf.org/html/rfc7628#section-3.1. The key-value pairs are + * copied. + * @param extension_size the number of SASL extension keys plus values, + * which must be a non-negative multiple of 2. + * @param errstr A human readable error string (nul-terminated) is written to + * this location that must be of at least \p errstr_size bytes. + * The \p errstr is only written in case of error. + * @param errstr_size Writable size in \p errstr. + * + * The SASL/OAUTHBEARER token refresh callback or event handler should invoke + * this method upon success. + * + * @returns \c RD_KAFKA_RESP_ERR_NO_ERROR on success, otherwise \p errstr set + * and:
+ * \c RD_KAFKA_RESP_ERR__INVALID_ARG if any of the arguments are + * invalid;
+ * \c RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED if SASL/OAUTHBEARER is not + * supported by this build;
+ * \c RD_KAFKA_RESP_ERR__STATE if SASL/OAUTHBEARER is supported but is + * not configured as the client's authentication mechanism.
+ * + * @sa rd_kafka_oauthbearer_set_token() + * @sa rd_kafka_share_oauthbearer_set_token_failure() + * @sa rd_kafka_conf_set_oauthbearer_token_refresh_cb() + */ +RD_EXPORT +rd_kafka_resp_err_t +rd_kafka_share_oauthbearer_set_token(rd_kafka_share_t *rkshare, + const char *token_value, + int64_t md_lifetime_ms, + const char *md_principal_name, + const char **extensions, + size_t extension_size, + char *errstr, + size_t errstr_size); + +/** + * @brief SASL/OAUTHBEARER token refresh failure indicator for a share + * consumer. + * + * @param rkshare Share consumer instance. + * @param errstr mandatory human readable error reason for failing to acquire + * a token. + * + * The SASL/OAUTHBEARER token refresh callback or event handler should invoke + * this method upon failure. + * + * @returns \c RD_KAFKA_RESP_ERR_NO_ERROR on success, otherwise:
+ * \c RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED if SASL/OAUTHBEARER is not + * supported by this build;
+ * \c RD_KAFKA_RESP_ERR__STATE if SASL/OAUTHBEARER is supported but is + * not configured as the client's authentication mechanism,
+ * \c RD_KAFKA_RESP_ERR__INVALID_ARG if no error string is supplied. + * + * @sa rd_kafka_oauthbearer_set_token_failure() + * @sa rd_kafka_share_oauthbearer_set_token() + * @sa rd_kafka_conf_set_oauthbearer_token_refresh_cb() + */ +RD_EXPORT +rd_kafka_resp_err_t +rd_kafka_share_oauthbearer_set_token_failure(rd_kafka_share_t *rkshare, + const char *errstr); + /** * @brief Flags for rd_kafka_destroy_flags() */ @@ -5020,6 +5229,906 @@ rd_kafka_resp_err_t rd_kafka_purge(rd_kafka_t *rk, int purge_flags); /**@}*/ + +/** + * @name Share consumer (Queues for Kafka) + * @brief Cooperative, queue-style consumption with a share group (KIP-932) + * @{ + * + * @warning **Preview feature.** The share consumer is provided as a preview: + * the public interfaces may change in a future release and it is not + * recommended for production use. It is currently available through + * the C API only; there is no C++ wrapper yet. See the "Current + * limitations" section below. + * + * @par Overview + * A share consumer lets multiple members of a *share group* cooperatively + * consume records from the same topic partitions, providing queue-like + * semantics on top of Kafka. This differs from consumers in a consumer group + * (rd_kafka_subscribe(), et.al.) where each partition is assigned to exactly + * one member of the group at a time. In a share group a single partition may + * be consumed by several members at once, and individual records — rather than + * the committed offset — are the unit of progress: each delivered record is + * *acquired* by a member under a time-limited acquisition lock, processed, and + * then *acknowledged*. The lock duration is a broker/group setting + * (\c group.share.record.lock.duration.ms, 30 seconds by default) and is not + * configured on this client. A record that is not acknowledged before its lock + * expires, or that is released, becomes available again and may be redelivered + * (possibly to a different member). A record thus moves through the states + * *available* -> *acquired* -> *acknowledged*; a rejected record, or one that + * exceeds the broker's delivery-count limit, becomes *archived* and is no + * longer delivered. + * + * For a conceptual overview of share groups and Queues for Kafka, see KIP-932 + * (https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka) + * and the Confluent share consumer documentation + * (https://docs.confluent.io/platform/current/clients/share-consumers.html). + * + * A share consumer is created with rd_kafka_share_consumer_new() rather than + * rd_kafka_new(), and is represented by the opaque \c rd_kafka_share_t handle. + * Its lifecycle mirrors the regular consumer: + * subscribe -> poll/acknowledge (loop) -> close -> destroy. + * + * @par Broker compatibility + * Requires a broker with share groups enabled. Share groups are available + * since Apache Kafka 4.2.0. The set of partitions assigned to each member is + * decided entirely by the broker (driven by the share group heartbeat); there + * is no client-side rebalance callback or assign() step. + * + * @par Configuration + * Configure the handle through the normal \c rd_kafka_conf_t interface before + * calling rd_kafka_share_consumer_new(). \c group.id is required. + * \c share.acknowledgement.mode is optional and selects the acknowledgement + * mode (see below); when unset it defaults to \c implicit. It is the only + * share-specific client property: other share-group settings (acquisition-lock + * duration, delivery-count limit, session/heartbeat timeouts, isolation level, + * acquire mode and offset reset) are broker/group settings and are not exposed + * by this client. + * Several regular-consumer properties are not applicable to share consumers + * (for example \c partition.assignment.strategy, \c enable.auto.commit, + * \c isolation.level, \c enable.partition.eof and the per-partition fetch + * queue tuning). A few network-related defaults also differ for share + * consumers: \c receive.message.max.bytes, \c connections.max.idle.ms, + * \c reconnect.backoff.ms and \c reconnect.backoff.max.ms. See + * \ref CONFIGURATION.md, where such properties are marked, for the + * authoritative list and the share-consumer default values. + * + * @par Subscription + * Subscribe to a set of topics with rd_kafka_share_subscribe() (exact topic + * names only; wildcard/regex subscriptions are not supported), inspect the + * current subscription with rd_kafka_share_subscription(), and clear it with + * rd_kafka_share_unsubscribe(). + * + * @par Polling + * rd_kafka_share_poll() returns a *batch* of messages in a single call (unlike + * the regular single-message poll), as an opaque \c rd_kafka_messages_t handle. + * Iterate it with rd_kafka_messages_count() and rd_kafka_messages_get(), and + * release it with rd_kafka_messages_destroy(). Record-level errors are surfaced + * as individual messages with a non-zero \c rd_kafka_message_t.err field + * (topic, partition and offset remain valid), so the application should check + * each message's \c err before treating it as data. \c max.poll.records bounds + * the batch size; in this preview it is a soft bound and the actual count may + * differ. + * + * @par Acknowledgement types + * Each acquired record is acknowledged with one of + * #rd_kafka_share_AcknowledgeType_t: + * - #RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT — processed successfully. + * - #RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE — not processed; make available + * again for redelivery. + * - #RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT — do not deliver again. + * @note RENEW (acquisition-lock renewal) is not yet available; see + * "Current limitations" below. + * The number of times a record has been delivered is available via + * rd_kafka_message_delivery_count(), which can be used to detect and reject + * "poison" records after a threshold. The broker also enforces its own maximum + * delivery count (\c group.share.delivery.count.limit, default 5); once a + * record exceeds it the broker archives the record and stops redelivering it. + * + * @par Acknowledgement modes + * The acknowledgement mode is fixed at creation by + * \c share.acknowledgement.mode: + * - \b implicit (default): the application does not call the acknowledge APIs. + * All records returned by a poll are automatically accepted (ACCEPT) when + * the next rd_kafka_share_poll() / rd_kafka_share_commit_sync() / + * rd_kafka_share_commit_async() is issued. + * - \b explicit: the application must acknowledge every record returned by a + * poll, using rd_kafka_share_acknowledge(), + * rd_kafka_share_acknowledge_type() or rd_kafka_share_acknowledge_offset(), + * before the next poll. If any record from the previous batch is still + * unacknowledged, rd_kafka_share_poll() returns + * RD_KAFKA_RESP_ERR__STATE. Records you have acknowledged but not yet + * committed are also committed when the consumer is closed (unless it is + * destroyed with RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE). + * + * @par Committing acknowledgements + * Acknowledgements are sent to the broker as part of the next poll, or + * explicitly with rd_kafka_share_commit_async() (fire-and-forget) or + * rd_kafka_share_commit_sync() (blocks for the broker replies and reports + * per-partition results). The acknowledgement-commit callback registered with + * rd_kafka_share_set_acknowledgement_commit_cb() is invoked with the outcome + * for each partition. Failed acknowledgements are not retried automatically; + * the error is reported to the application. + * + * @par Example: implicit acknowledgement (default) + * @code + * rd_kafka_share_t *rkshare = rd_kafka_share_consumer_new(conf, errstr, + * sizeof(errstr)); + * rd_kafka_share_subscribe(rkshare, topics); + * while (run) { + * rd_kafka_messages_t *batch = NULL; + * rd_kafka_error_t *error = rd_kafka_share_poll(rkshare, 500, &batch); + * if (error) { + * // handle/log error + * rd_kafka_error_destroy(error); + * } else { + * size_t cnt = rd_kafka_messages_count(batch); + * for (size_t i = 0 ; i < cnt ; i++) { + * rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, i); + * if (rkm->err) { + * // Record-level error. The acknowledgement has already + * // been applied internally based on the error type: + * // corrupt/CRC (RD_KAFKA_RESP_ERR__BAD_MSG) and + * // unsupported (RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED) batches + * // are REJECTed; decompression failures are RELEASEd for + * // redelivery. In implicit mode the application only needs + * // to inspect rkm->err. + * } else { + * // process rkm (auto-accepted on next poll) + * } + * } + * } + * rd_kafka_messages_destroy(batch); // NULL-safe + * } + * rd_kafka_share_consumer_close(rkshare); + * rd_kafka_share_destroy(rkshare); + * @endcode + * + * @par Example: explicit acknowledgement + * @code + * // Created with conf "share.acknowledgement.mode" = "explicit". + * rd_kafka_messages_t *batch = NULL; + * rd_kafka_error_t *error = rd_kafka_share_poll(rkshare, 500, &batch); + * if (!error) { + * size_t cnt = rd_kafka_messages_count(batch); + * for (size_t i = 0 ; i < cnt ; i++) { + * rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, i); + * if (process(rkm) == OK) + * // Processed successfully. + * rd_kafka_share_acknowledge_type( + * rkshare, rkm, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + * else if (is_temporary(rkm)) + * // Temporary failure: release for redelivery (possibly to + * // another member) so it can be retried. + * rd_kafka_share_acknowledge_type( + * rkshare, rkm, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + * else + * // Permanent failure (e.g. a "poison" record): reject so it + * // is not delivered again. + * rd_kafka_share_acknowledge_type( + * rkshare, rkm, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + * } + * // Every record must be acknowledged before the next poll. + * } + * rd_kafka_error_destroy(error); // NULL-safe + * rd_kafka_messages_destroy(batch); + * @endcode + * + * For complete runnable programs see \c examples/share_consumer.c, + * \c examples/share_consumer_commit_sync.c and + * \c examples/share_consumer_commit_async.c. + * + * @par Thread safety + * The share consumer handle is **not thread-safe by design**: a single + * \c rd_kafka_share_t handle must not be used concurrently from multiple + * threads. This matches the share consumer design in KIP-932. Use one handle + * per thread, or serialise all access to a handle. Concurrent use is detected + * on a best-effort basis and rejected with RD_KAFKA_RESP_ERR__CONFLICT. Unlike + * the regular consumer there is no wakeup mechanism in this preview, so a + * blocking poll/commit ends only when its timeout expires. + * + * @par Closing + * Close the consumer with rd_kafka_share_consumer_close() (or the asynchronous + * rd_kafka_share_consumer_close_queue()), which sends any pending + * acknowledgements and leaves the share group, then free the handle with + * rd_kafka_share_destroy(). Close takes no timeout argument in this preview and + * is bounded internally to roughly \c socket.timeout.ms. + * + * @par Current limitations (preview) + * - No strict per-poll record limit: \c max.poll.records is a soft bound. + * - Acknowledgement types are limited to ACCEPT / RELEASE / REJECT; there is + * no acquisition-lock renewal, so a long-running handler cannot extend a + * record's lock and the record may be redelivered. + * - No wakeup API; blocking calls end only at their timeout. + * - No share-group admin operations and no share-consumer client metrics APIs. + * - Topics are resolved by id; auto topic creation is not performed. + * - Some app-visible error codes differ from other clients because librdkafka + * generates certain failures locally (for example + * RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER, RD_KAFKA_RESP_ERR__TRANSPORT, + * RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH). + * + * @sa rd_kafka_share_consumer_new() + * @sa rd_kafka_share_set_acknowledgement_commit_cb() + * @sa rd_kafka_share_subscribe() + * @sa rd_kafka_share_unsubscribe() + * @sa rd_kafka_share_subscription() + * @sa rd_kafka_share_poll() + * @sa rd_kafka_messages_count() + * @sa rd_kafka_messages_get() + * @sa rd_kafka_messages_destroy() + * @sa rd_kafka_message_delivery_count() + * @sa rd_kafka_share_acknowledge() + * @sa rd_kafka_share_acknowledge_type() + * @sa rd_kafka_share_acknowledge_offset() + * @sa rd_kafka_share_commit_sync() + * @sa rd_kafka_share_commit_async() + * @sa rd_kafka_share_consumer_close() + * @sa rd_kafka_share_consumer_close_queue() + * @sa rd_kafka_share_consumer_closed() + * @sa rd_kafka_share_destroy() + * @sa rd_kafka_share_destroy_flags() + * @sa rd_kafka_share_set_log_queue() + * @sa rd_kafka_share_sasl_background_callbacks_enable() + * @sa rd_kafka_share_sasl_set_credentials() + * @sa rd_kafka_share_oauthbearer_set_token() + * @sa rd_kafka_share_oauthbearer_set_token_failure() + */ + +/** + * @enum rd_kafka_share_AcknowledgeType_t + * @brief Share consumer acknowledgement types (exposed to Public APIs). + */ +typedef enum rd_kafka_share_AcknowledgeType_s { + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT = 1, /**< Accept the message */ + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE = 2, /**< Release the message + * for redelivery */ + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT = 3 /**< Reject the message */ +} rd_kafka_share_AcknowledgeType_t; + +/** + * @brief Share consumer partition offsets for a single offset range. + * + * Contains information about an acknowledged offset range including + * topic, partition, offset range, and any error. + */ +typedef struct rd_kafka_share_partition_offsets_s + rd_kafka_share_partition_offsets_t; + +/** + * @brief List of share consumer partition offsets. + * + * Contains multiple partition offset entries. + */ +typedef struct rd_kafka_share_partition_offsets_list_s + rd_kafka_share_partition_offsets_list_t; + +/** + * @brief Get the number of partitions in the offsets list. + * @param list Partition offsets list. + * @returns The number of partitions in the list. + */ +RD_EXPORT size_t rd_kafka_share_partition_offsets_list_count( + const rd_kafka_share_partition_offsets_list_t *list); + +/** + * @brief Get a partition offsets entry at the specified index. + * @param list Partition offsets list. + * @param index Zero-based index of the partition to retrieve. + * @returns Pointer to the partition offsets entry + */ +RD_EXPORT const rd_kafka_share_partition_offsets_t * +rd_kafka_share_partition_offsets_list_get( + const rd_kafka_share_partition_offsets_list_t *list, + size_t index); + +/** + * @brief Destroy the partition offsets list. + * @param list Partition offsets list to destroy. + * + * @remark Only call this if you have ownership of the list. + * The list passed to the callback is owned by librdkafka + * and will be destroyed automatically after the callback returns. + */ +RD_EXPORT void rd_kafka_share_partition_offsets_list_destroy( + rd_kafka_share_partition_offsets_list_t *list); + +/** + * @brief Get the topic partition from a partition offsets entry. + * @param partition_offsets The partition offsets entry. + * @returns Pointer to the topic partition (valid for the lifetime of the list). + */ +RD_EXPORT const rd_kafka_topic_partition_t * +rd_kafka_share_partition_offsets_partition( + const rd_kafka_share_partition_offsets_t *partition_offsets); + +/* + * TODO KIP-932: Recheck name once public interfaces are finalized. + */ +/** + * @brief Get the offsets array from a partition offsets entry. + * @param partition_offsets The partition offsets entry. + * @returns Array of acknowledged offsets (valid for the lifetime of the list). + */ +RD_EXPORT const int64_t *rd_kafka_share_partition_offsets_offsets( + const rd_kafka_share_partition_offsets_t *partition_offsets); + +/** + * @brief Get the number of offsets in a partition offsets entry. + * @param partition_offsets The partition offsets entry. + * @returns The number of offsets in the array. + */ +RD_EXPORT size_t rd_kafka_share_partition_offsets_offsets_cnt( + const rd_kafka_share_partition_offsets_t *partition_offsets); + + +/* + * TODO KIP-932: understand whether all share consumer APIs need to be gated + * once a fatal error has been raised on the handle, and check what the Java + * client does in the fatal-error case. + */ +/** + * @brief Creates a new share consumer instance. + * + * \p conf is an optional struct created with `rd_kafka_conf_new()` that will + * be used instead of the default configuration. + * The \p conf object is freed by this function on success and must not be used + * or destroyed by the application subsequently. + * See `rd_kafka_conf_set()` et.al for more information. + * + * \p errstr must be a pointer to memory of at least size \p errstr_size where + * `rd_kafka_share_consumer_new()` may write a human readable error message + * in case the creation of a new handle fails. In which case the function + * returns NULL. + * + * @param conf Configuration. The \p conf object is freed by this function + * on success and must not be used or destroyed by the application + * subsequently. + * @param errstr A human readable error string is written to this location + * if the function fails. + * @param errstr_size The size of the \p errstr buffer. + * + * @returns The share consumer handle on success or NULL on error + * (see \p errstr). + * + * @remark \c group.id is required. \c share.acknowledgement.mode is optional + * and defaults to \c implicit. See \ref CONFIGURATION.md for + * share-consumer defaults and the properties that are not supported + * for share consumers. + * + * @sa rd_kafka_share_subscribe() + * @sa rd_kafka_share_destroy() + */ +RD_EXPORT +rd_kafka_share_t *rd_kafka_share_consumer_new(rd_kafka_conf_t *conf, + char *errstr, + size_t errstr_size); + + +/** + * @brief Set acknowledgement commit callback at runtime. + * + * Sets a callback to be invoked when acknowledgements sent to the broker + * have been completed (either successfully or with an error). + * + * This callback can be changed at any time. The currently set callback + * will be used when processing acknowledgement responses, not the callback + * that was active when acknowledgements were sent. + * + * The callback is invoked once per partition with the acknowledgement result + * for that partition. + * + * @param rkshare Share consumer handle. + * @param share_acknowledgement_commit_cb Callback function, or NULL to clear. + * @param opaque Application opaque passed to callback. + * + * @remark Cannot be called from within the acknowledgement callback itself. + * @remark Replaces any previously set callback. + * @remark Registration is asynchronous - there is a brief window after + * calling this function during which the new registration state + * is not yet visible to the main thread. + * + * @returns NULL on success, otherwise an rd_kafka_error_t* (which must be freed + * with rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer has been closed, or the call is made from inside the + * acknowledgement-commit callback, etc. + */ +RD_EXPORT rd_kafka_error_t *rd_kafka_share_set_acknowledgement_commit_cb( + rd_kafka_share_t *rkshare, + void (*share_acknowledgement_commit_cb)( + rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque), + void *opaque); + + +/** + * @brief Subscribe a share consumer to a set of topics. + * + * Wildcard (regex) topics are not supported. Only the \c .topic field + * is used in \p topics; all other fields are ignored. + * + * @param rkshare Share consumer instance. + * @param topics Topics to subscribe to. + * + * @remark This call is asynchronous and returns immediately. The background + * thread sends a ShareGroupHeartbeat to the broker, which assigns + * partitions to the consumer. Partition assignment is entirely + * broker-driven; there is no client-side rebalance callback or + * assign() step. + * + * @remark Topic names are forwarded verbatim to the broker via the share + * group heartbeat. The broker validates them; unavailable or + * unauthorized topics will surface as errors via + * rd_kafka_share_poll(). + * + * @remark If \p topics is empty (cnt == 0), the call is equivalent to + * rd_kafka_share_unsubscribe(): the subscription is cleared and + * RD_KAFKA_RESP_ERR_NO_ERROR is returned. + * + * @returns + * - RD_KAFKA_RESP_ERR_NO_ERROR on success, or when \p topics is empty + * (treated as unsubscribe). + * - RD_KAFKA_RESP_ERR__INVALID_ARG if \p topics contains empty topic names or + * duplicate entries. + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer has been (or is being) closed, or the call is made from inside + * the acknowledgement-commit callback, etc. + * - RD_KAFKA_RESP_ERR__FATAL if the consumer has raised a fatal error. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_share_subscribe(rd_kafka_share_t *rkshare, + const rd_kafka_topic_partition_list_t *topics); + + +/** + * @brief Unsubscribe from the current subscription set. + * + * Clears the subscription; the broker removes the member from the share group. + * + * @param rkshare Share consumer instance. + * + * @returns + * - RD_KAFKA_RESP_ERR_NO_ERROR on success. + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer has been (or is being) closed, or the call is made from inside + * the acknowledgement-commit callback, etc. + * - Otherwise, an error code from the underlying unsubscribe. + */ +RD_EXPORT +rd_kafka_resp_err_t rd_kafka_share_unsubscribe(rd_kafka_share_t *rkshare); + + +/** + * @brief Returns the current topic subscription + * + * @param rkshare Share consumer instance. + * @param topics Output pointer to a newly allocated topic list (possibly + * empty) on success. + * + * @returns + * - RD_KAFKA_RESP_ERR_NO_ERROR on success, in which case \p topics is updated + * to point to a newly allocated topic list (possibly empty). + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer has been (or is being) closed, or the call is made from inside + * the acknowledgement-commit callback, etc. + * - Otherwise, an error code from the underlying subscription query. + * + * @remark The application is responsible for calling + * rd_kafka_topic_partition_list_destroy on the returned list. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_share_subscription(rd_kafka_share_t *rkshare, + rd_kafka_topic_partition_list_t **topics); + + +/** + * @brief Poll the share consumer for a batch of messages. It is different from + * normal consumer poll where a single record is returned, this API + * returns a batch of messages in a single call. + * + * @param rkshare Share consumer instance. + * @param timeout_ms Maximum time to block waiting for messages. + * @param rkmessages Output pointer to a newly-allocated message list (callee + * allocates). On return: + * - On success with at least one message: \c *rkmessages + * points to an \c rd_kafka_messages_t handle owned by the + * caller, who must release it with + * rd_kafka_messages_destroy(). + * - On error or timeout with no messages: + * \c *rkmessages is set to NULL. + * \c rd_kafka_messages_destroy(NULL) is a safe no-op so + * callers may unconditionally destroy. + * + * Use rd_kafka_messages_count() and rd_kafka_messages_get() to access the + * returned messages. + * + * @returns NULL on success (including a timeout with no messages, in which + * case \c *rkmessages is NULL), otherwise an \c rd_kafka_error_t + * (which must be freed with rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer is not subscribed, it has been closed, or (explicit mode) records + * from the previous poll have not all been acknowledged, etc. + * - RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED if the caller is not + * authorized to read one of the subscribed topics or the share group. + * - A fatal-flagged error (rd_kafka_error_is_fatal() is true) carrying the + * specific error code, if a fatal error has been raised on the handle. + * - Another error code for any other failure encountered while fetching. + * + * @note \c *rkmessages must point to NULL on entry. The function + * unconditionally overwrites \c *rkmessages, so any non-NULL value it held on + * entry is overwritten and leaked. + * + * @remark A single call returns either an error or messages, never both: on + * failure a non-NULL error is returned and \c *rkmessages is NULL; on + * success the error is NULL and \c *rkmessages points to the batch. + * On a timeout with no messages both are NULL. + * + * @remark Record-level errors are delivered as messages with a non-zero + * \c rd_kafka_message_t.err field (topic/partition/offset intact); + * the application should inspect each message's \c err. + * + * @remark In explicit acknowledgement mode every record returned by the + * previous poll must be acknowledged before calling this function + * again. + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_poll(rd_kafka_share_t *rkshare, + int timeout_ms, + rd_kafka_messages_t **rkmessages); + + +/** + * @brief Acknowledge a message with ACCEPT type. + * + * This is equivalent to calling rd_kafka_share_acknowledge_type() with + * RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT. + * + * @param rkshare Share consumer handle. + * @param rkmessage Message to acknowledge. + * + * @returns + * - RD_KAFKA_RESP_ERR_NO_ERROR on success. + * - RD_KAFKA_RESP_ERR__INVALID_ARG on invalid input parameters (e.g. rkmessage + * is NULL). + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer is not in explicit acknowledgement mode, it has been closed, or + * the message is unknown, etc. + */ +RD_EXPORT +rd_kafka_resp_err_t +rd_kafka_share_acknowledge(rd_kafka_share_t *rkshare, + const rd_kafka_message_t *rkmessage); + +/** + * @brief Acknowledge a message with specified acknowledgement type. + * + * @param rkshare Share consumer handle. + * @param rkmessage Message to acknowledge. + * @param type Acknowledgement type (ACCEPT, RELEASE, or REJECT). + * + * @returns + * - RD_KAFKA_RESP_ERR_NO_ERROR on success. + * - RD_KAFKA_RESP_ERR__INVALID_ARG on invalid input parameters (e.g. rkmessage + * is NULL or \p type is out of range). + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer is not in explicit acknowledgement mode, it has been closed, or + * the message is unknown, etc. + */ +RD_EXPORT +rd_kafka_resp_err_t +rd_kafka_share_acknowledge_type(rd_kafka_share_t *rkshare, + const rd_kafka_message_t *rkmessage, + rd_kafka_share_AcknowledgeType_t type); + +/* + * TODO KIP-932: Check if it should only be allowed for error cases + * like Java. + */ +/** + * @brief Acknowledge an offset directly with specified acknowledgement type. + * + * @param rkshare Share consumer handle. + * @param topic Topic name. + * @param partition Partition id. + * @param offset Offset to acknowledge. + * @param type Acknowledgement type (ACCEPT, RELEASE, or REJECT). + * + * @returns + * - RD_KAFKA_RESP_ERR_NO_ERROR on success. + * - RD_KAFKA_RESP_ERR__INVALID_ARG on invalid input parameters (e.g. topic is + * NULL, partition < 0, offset < 0, or \p type is out of range). + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer is not in explicit acknowledgement mode, it has been closed, or + * the offset is unknown, etc. + */ +RD_EXPORT +rd_kafka_resp_err_t +rd_kafka_share_acknowledge_offset(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t offset, + rd_kafka_share_AcknowledgeType_t type); + + +/** + * @brief Asynchronously commit all pending acknowledgements. + * + * Sends all pending acknowledgements (from rd_kafka_share_acknowledge* + * calls) to their respective brokers without fetching new records. + * This function returns immediately — the acknowledgements are sent + * asynchronously via the broker threads. + * + * The per-partition outcome is delivered to the acknowledgement-commit + * callback registered with rd_kafka_share_set_acknowledgement_commit_cb(). + * Failed acknowledgements are not retried automatically. + * + * @param rkshare Share consumer instance. + * + * @returns NULL on success, otherwise an \c rd_kafka_error_t* (which must be + * freed with rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer has been (or is being) closed, or the call is made from inside + * the acknowledgement-commit callback, etc. + * + * @sa rd_kafka_share_set_acknowledgement_commit_cb() + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_commit_async(rd_kafka_share_t *rkshare); + +/** + * @brief Synchronously commit all pending acknowledgements. + * + * Sends all pending acknowledgements (from rd_kafka_share_acknowledge* + * calls) to their respective brokers and blocks until all broker + * replies are received or the timeout expires. + * + * In implicit ack mode, all ACQUIRED records from the previous poll + * are first converted to ACCEPT before committing. + * + * Failed acknowledgements are not retried automatically; the per-partition + * error is reported through \p partitions. + * + * @param rkshare Share consumer instance. + * @param timeout_ms Timeout in milliseconds. + * @param partitions [out] Per-partition results. Each partition's err + * field contains the result for that partition — for example + * RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT (on timeout), + * RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER, + * RD_KAFKA_RESP_ERR__TRANSPORT, or a share-session error such + * as RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH, etc. + * The caller must destroy the returned list with + * rd_kafka_topic_partition_list_destroy(). + * Set to NULL if no acks were pending. + * + * @returns NULL on success, otherwise an \c rd_kafka_error_t* (which must be + * freed with rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer has been (or is being) closed, or the call is made from inside + * the acknowledgement-commit callback, etc. + * + * A non-NULL error reflects a call-level failure; per-partition + * failures are reported through \p partitions. + */ +RD_EXPORT +rd_kafka_error_t * +rd_kafka_share_commit_sync(rd_kafka_share_t *rkshare, + int timeout_ms, + rd_kafka_topic_partition_list_t **partitions); + + +/** + * @brief Close the share consumer. + * + * This call will block until the consumer has committed all the pending + * acknowledgments and left the consumer group. The maximum blocking time is + * roughly limited to socket.timeout.ms. + * + * @param rkshare Share consumer instance. + * + * @returns NULL on success, otherwise an error object (which must be freed + * with rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer is already closed or closing, or the call is made from inside + * the acknowledgement-commit callback, etc. + * - RD_KAFKA_RESP_ERR__FATAL if the consumer has raised a fatal error. + * + * @remark The application still needs to call rd_kafka_share_destroy() after + * this call finishes to clean up the underlying handle resources. + * + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_consumer_close(rd_kafka_share_t *rkshare); + +/** + * @brief Asynchronously close the share consumer. + * + * Performs the same actions as rd_kafka_share_consumer_close() but in a + * background thread. + * + * Callbacks (etc) will be forwarded to the + * application-provided \p rkqu. The application must poll/serve this queue + * until rd_kafka_share_consumer_closed() returns true. + * + * @param rkshare Share consumer instance. + * @param rkqu Queue to forward close callbacks to. + * + * @returns NULL on success, otherwise an error object (which must be freed + * with rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently inside a + * share consumer API. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * consumer is already closed or closing, or the call is made from inside + * the acknowledgement-commit callback, etc. + * + * @sa rd_kafka_share_consumer_closed() + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_consumer_close_queue(rd_kafka_share_t *rkshare, + rd_kafka_queue_t *rkqu); + + +/** + * @brief Check whether the share consumer has finished closing. + * + * @param rkshare Share consumer instance. + * + * @returns 1 if the consumer is closed, else 0. + * + * Should be used in conjunction with rd_kafka_share_consumer_close_queue() to + * know when the consumer has been closed. + * + * @sa rd_kafka_share_consumer_close_queue() + */ +RD_EXPORT +int rd_kafka_share_consumer_closed(rd_kafka_share_t *rkshare); + + +/** + * @brief Destroy the share consumer instance and free all associated resources. + * + * @param rkshare Share consumer instance. + * + * @returns NULL on success, otherwise an error object (which must be freed with + * rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently using the + * share consumer. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * call is made from inside the acknowledgement-commit callback, etc. + * + * In the error case the instance is NOT destroyed; the application + * must ensure no other thread is using it and call destroy again. + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_destroy(rd_kafka_share_t *rkshare); + +/** + * @brief Destroy the share consumer instance according to specified destroy + * flags. + * + * @param rkshare Share consumer instance. + * + * @param flags Destroy flags (see RD_KAFKA_DESTROY_F_* defines). + * + * @returns NULL on success, otherwise an error object (which must be freed with + * rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__CONFLICT if another thread is concurrently using the + * share consumer. + * - RD_KAFKA_RESP_ERR__STATE if called in an invalid state — for example the + * call is made from inside the acknowledgement-commit callback, etc. + * + * In the error case the instance is NOT destroyed. + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_destroy_flags(rd_kafka_share_t *rkshare, + int flags); + + +/** + * @brief Forward a share consumer's internal log queue onto another queue + * for serving with one of the ..poll() calls. + * + * Thin wrapper around rd_kafka_set_log_queue() for use with + * rd_kafka_share_t handles. Required when `log.queue=true` is + * configured on a share consumer — without this call the share + * consumer's internal log queue is never drained and log_cb never + * fires. + * + * @param rkshare Share consumer instance. + * @param rkqu Queue to forward logs to. If NULL the logs are forwarded + * to the share consumer's internal rk_rep, which is drained + * by rd_kafka_share_poll() / + * rd_kafka_share_commit_sync() / rd_kafka_share_commit_async() + * on the application's poll thread. + * + * @remark The configuration property `log.queue` MUST also be set to true. + * + * @remark librdkafka maintains its own reference to the provided queue. + * + * @returns NULL on success, otherwise an error object (which must be freed with + * rd_kafka_error_destroy()) with one of: + * - RD_KAFKA_RESP_ERR__INVALID_ARG if \p rkshare is NULL or uninitialized. + * - RD_KAFKA_RESP_ERR__NOT_CONFIGURED if `log.queue` is not set to true. + * + * @sa rd_kafka_set_log_queue + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_set_log_queue(rd_kafka_share_t *rkshare, + rd_kafka_queue_t *rkqu); + + +/** + * @brief Enable SASL background callbacks for a share consumer. + * + * This is a convenience wrapper around + * rd_kafka_sasl_background_callbacks_enable() for share consumers. + * It forwards the SASL queue to the background thread so that + * OAUTHBEARER token refresh callbacks are served automatically. + * + * @param rkshare Share consumer instance. + * + * @returns NULL on success or an error object on failure. + * + * @sa rd_kafka_sasl_background_callbacks_enable() + * @sa rd_kafka_queue_get_sasl() + * @sa rd_kafka_conf_set_oauthbearer_token_refresh_cb() + */ +RD_EXPORT +rd_kafka_error_t * +rd_kafka_share_sasl_background_callbacks_enable(rd_kafka_share_t *rkshare); + + +/** + * @brief Sets SASL credentials used for SASL PLAIN and SCRAM mechanisms by + * this share consumer. + * + * This function sets or resets the SASL username and password credentials + * used by this share consumer. The new credentials will be used the next time + * this client needs to authenticate to a broker. This function + * will not disconnect existing connections that might have been made using + * the old credentials. + * + * @remark This function only applies to the SASL PLAIN and SCRAM mechanisms. + * + * @param rkshare Share consumer instance. + * @param username New SASL username. + * @param password New SASL password. + * + * @returns NULL on success or an error object on failure. + * + * @sa rd_kafka_sasl_set_credentials() + */ +RD_EXPORT +rd_kafka_error_t *rd_kafka_share_sasl_set_credentials(rd_kafka_share_t *rkshare, + const char *username, + const char *password); +/**@}*/ + + + /** * @name Metadata API * @{ @@ -5315,9 +6424,9 @@ rd_kafka_consumer_group_type_name(rd_kafka_consumer_group_type_t type); * * @param name The group type name. * - * @remark The comparison is case-insensitive. - * * @return The group type value corresponding to the provided group type name. + * + * @remark The comparison is case-insensitive. */ RD_EXPORT rd_kafka_consumer_group_type_t diff --git a/lib/librdkafka-2.10.1/src/rdkafka_admin.c b/lib/librdkafka-2.15.0/src/rdkafka_admin.c similarity index 96% rename from lib/librdkafka-2.10.1/src/rdkafka_admin.c rename to lib/librdkafka-2.15.0/src/rdkafka_admin.c index b2671f3c821..0039627ffca 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_admin.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_admin.c @@ -31,6 +31,7 @@ #include "rdkafka_admin.h" #include "rdkafka_request.h" #include "rdkafka_aux.h" +#include "rdunittest.h" #include @@ -2318,7 +2319,7 @@ rd_kafka_DeleteTopicsResponse_parse(rd_kafka_op_t *rko_req, } /* #topics */ - rd_kafka_buf_read_i32(reply, &topic_cnt); + rd_kafka_buf_read_arraycnt(reply, &topic_cnt, RD_KAFKAP_TOPICS_MAX); if (topic_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args)) rd_kafka_buf_parse_fail( @@ -2342,6 +2343,7 @@ rd_kafka_DeleteTopicsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &ktopic); rd_kafka_buf_read_i16(reply, &error_code); + rd_kafka_buf_skip_tags(reply); /* For non-blocking DeleteTopicsRequests the broker * will returned REQUEST_TIMED_OUT for topics @@ -2591,7 +2593,7 @@ rd_kafka_CreatePartitionsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_op_throttle_time(rkb, rk->rk_rep, Throttle_Time); /* #topics */ - rd_kafka_buf_read_i32(reply, &topic_cnt); + rd_kafka_buf_read_arraycnt(reply, &topic_cnt, RD_KAFKAP_PARTITIONS_MAX); if (topic_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args)) rd_kafka_buf_parse_fail( @@ -2618,6 +2620,8 @@ rd_kafka_CreatePartitionsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &ktopic); rd_kafka_buf_read_i16(reply, &error_code); rd_kafka_buf_read_str(reply, &error_msg); + rd_kafka_buf_skip_tags(reply); + /* For non-blocking CreatePartitionsRequests the broker * will returned REQUEST_TIMED_OUT for topics @@ -4066,11 +4070,11 @@ rd_kafka_DeleteRecordsResponse_parse(rd_kafka_op_t *rko_req, RD_KAFKA_TOPIC_PARTITION_FIELD_END}; offsets = rd_kafka_buf_read_topic_partitions( reply, rd_false /*don't use topic_id*/, rd_true, 0, fields); + if (!offsets) rd_kafka_buf_parse_fail(reply, "Failed to parse topic partitions"); - rko_result = rd_kafka_admin_result_new(rko_req); rd_list_init(&rko_result->rko_u.admin_result.results, 1, rd_kafka_topic_partition_list_destroy_free); @@ -4760,7 +4764,7 @@ rd_kafka_DeleteGroupsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_throttle_time(reply); /* #group_error_codes */ - rd_kafka_buf_read_i32(reply, &group_cnt); + rd_kafka_buf_read_arraycnt(reply, &group_cnt, RD_KAFKAP_GROUPS_MAX); if (group_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args)) rd_kafka_buf_parse_fail( @@ -4781,6 +4785,7 @@ rd_kafka_DeleteGroupsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &kgroup); rd_kafka_buf_read_i16(reply, &error_code); + rd_kafka_buf_skip_tags(reply); groupres = rd_kafka_group_result_new( kgroup.str, RD_KAFKAP_STR_LEN(&kgroup), NULL, @@ -5450,6 +5455,8 @@ rd_kafka_CreateAclsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &error_msg); + rd_kafka_buf_skip_tags(reply); + if (error_code) { if (RD_KAFKAP_STR_LEN(&error_msg) == 0) errstr = (char *)rd_kafka_err2str(error_code); @@ -5620,6 +5627,8 @@ rd_kafka_DescribeAclsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &khost); rd_kafka_buf_read_i8(reply, &operation); rd_kafka_buf_read_i8(reply, &permission_type); + rd_kafka_buf_skip_tags(reply); + RD_KAFKAP_STR_DUPA(&principal, &kprincipal); RD_KAFKAP_STR_DUPA(&host, &khost); @@ -5653,6 +5662,8 @@ rd_kafka_DescribeAclsResponse_parse(rd_kafka_op_t *rko_req, rd_list_add(&rko_result->rko_u.admin_result.results, acl); } + + rd_kafka_buf_skip_tags(reply); } *rko_resultp = rko_result; @@ -6032,9 +6043,9 @@ rd_kafka_UserScramCredentialUpsertion_new(const char *username, alteration->alteration.upsertion.salt = rd_kafkap_bytes_new(salt, salt_size); } else { -#if WITH_SSL && OPENSSL_VERSION_NUMBER >= 0x10101000L +#if HAVE_SECURE_RAND_BYTES unsigned char random_salt[64]; - if (RAND_priv_bytes(random_salt, sizeof(random_salt)) == 1) { + if (rd_rand_bytes(random_salt, sizeof(random_salt))) { alteration->alteration.upsertion.salt = rd_kafkap_bytes_new(random_salt, sizeof(random_salt)); @@ -6661,6 +6672,8 @@ rd_kafka_DeleteAclsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &khost); rd_kafka_buf_read_i8(reply, &operation); rd_kafka_buf_read_i8(reply, &permission_type); + rd_kafka_buf_skip_tags(reply); + RD_KAFKAP_STR_DUPA(&res_name, &kres_name); RD_KAFKAP_STR_DUPA(&principal, &kprincipal); RD_KAFKAP_STR_DUPA(&host, &khost); @@ -6718,6 +6731,8 @@ rd_kafka_DeleteAclsResponse_parse(rd_kafka_op_t *rko_req, (void *)matching_acl); } + rd_kafka_buf_skip_tags(reply); + rd_list_add(&rko_result->rko_u.admin_result.results, (void *)result_response); } @@ -7567,6 +7582,56 @@ rd_kafka_ListConsumerGroupsResponse_parse(rd_kafka_op_t *rko_req, return RD_KAFKA_RESP_ERR_NO_ERROR; } +/** + * @brief Compare two ConsumerGroupListing by group_id + */ +static int rd_kafka_ConsumerGroupListing_cmp_group_id(const void *a, + const void *b) { + const rd_kafka_ConsumerGroupListing_t *ga = a, *gb = b; + return strcmp(ga->group_id, gb->group_id); +} + +/** + * @brief Append groups from \p src to \p dst, skipping duplicates. + * + * Keeps \p dst sorted by group_id so that subsequent calls can use + * bsearch for O(log n) dedup lookups. + */ +static void +rd_kafka_ConsumerGroupListing_list_append_dedup(rd_list_t *dst, + const rd_list_t *src) { + const rd_kafka_ConsumerGroupListing_t *grp; + int j; + rd_list_t *to_add = NULL; + + if (!rd_list_cnt(src)) + return; + + /* Ensure dst is sorted for bsearch lookups. */ + if (!(dst->rl_flags & RD_LIST_F_SORTED)) + rd_list_sort(dst, rd_kafka_ConsumerGroupListing_cmp_group_id); + + /* Collect pointers to src groups not already in dst. + * dst is not modified here so the sorted flag stays valid. */ + RD_LIST_FOREACH(grp, src, j) { + if (rd_list_find(dst, grp, + rd_kafka_ConsumerGroupListing_cmp_group_id)) + continue; + if (!to_add) + to_add = rd_list_new(rd_list_cnt(src), NULL); + rd_list_add(to_add, (void *)grp); + } + + /* Copy new groups into dst and resort. */ + if (to_add) { + rd_list_copy_to(dst, to_add, + rd_kafka_ConsumerGroupListing_copy_opaque, + NULL); + rd_list_sort(dst, rd_kafka_ConsumerGroupListing_cmp_group_id); + rd_list_destroy(to_add); + } +} + /** @brief Merge the ListConsumerGroups response from a single broker * into the user response list. */ @@ -7597,7 +7662,7 @@ rd_kafka_ListConsumerGroups_response_merge(rd_kafka_op_t *rko_fanout, } if (!rko_partial->rko_err) { int new_valid_count, new_errors_count; - const rd_list_t *new_valid_list, *new_errors_list; + const rd_list_t *new_errors_list; /* Read the partial result and merge the valid groups * and the errors into the fanout parent result. */ newres = @@ -7606,11 +7671,8 @@ rd_kafka_ListConsumerGroups_response_merge(rd_kafka_op_t *rko_fanout, new_valid_count = rd_list_cnt(&newres->valid); new_errors_count = rd_list_cnt(&newres->errors); if (new_valid_count) { - new_valid_list = &newres->valid; - rd_list_grow(&res->valid, new_valid_count); - rd_list_copy_to( - &res->valid, new_valid_list, - rd_kafka_ConsumerGroupListing_copy_opaque, NULL); + rd_kafka_ConsumerGroupListing_list_append_dedup( + &res->valid, &newres->valid); } if (new_errors_count) { new_errors_list = &newres->errors; @@ -8286,6 +8348,7 @@ rd_kafka_DescribeConsumerGroupsResponse_parse(rd_kafka_op_t *rko_req, rd_kafka_buf_read_str(reply, &ClientHost); rd_kafka_buf_read_kbytes(reply, &MemberMetadata); rd_kafka_buf_read_kbytes(reply, &MemberAssignment); + rd_kafka_buf_skip_tags(reply); if (error != NULL) continue; @@ -8353,6 +8416,8 @@ rd_kafka_DescribeConsumerGroupsResponse_parse(rd_kafka_op_t *rko_req, authorized_operations, &operation_cnt); } + rd_kafka_buf_skip_tags(reply); + if (error == NULL) { grpdesc = rd_kafka_ConsumerGroupDescription_new( group_id, is_simple_consumer_group, &members, proto, @@ -9792,3 +9857,309 @@ void rd_kafka_ElectLeaders(rd_kafka_t *rk, } /**@}*/ + + +/** + * @name Unit tests + * @{ + */ + +/** + * @brief Helper to create a mock partial result op containing the given + * group_ids, for testing the merge function. + */ +static rd_kafka_op_t *ut_make_ListConsumerGroups_partial(const char **group_ids, + int group_cnt) { + rd_kafka_op_t *rko_partial; + rd_list_t valid, errors; + rd_kafka_ListConsumerGroupsResult_t *result; + int i; + + rko_partial = rd_kafka_op_new(RD_KAFKA_OP_ADMIN_RESULT); + rko_partial->rko_evtype = RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT; + rko_partial->rko_err = RD_KAFKA_RESP_ERR_NO_ERROR; + + rd_list_init(&valid, group_cnt, rd_kafka_ConsumerGroupListing_free); + rd_list_init(&errors, 0, rd_free); + + for (i = 0; i < group_cnt; i++) { + rd_list_add(&valid, rd_kafka_ConsumerGroupListing_new( + group_ids[i], rd_false, + RD_KAFKA_CONSUMER_GROUP_STATE_STABLE, + RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC)); + } + + result = rd_kafka_ListConsumerGroupsResult_new(&valid, &errors); + + rd_list_init(&rko_partial->rko_u.admin_result.results, 1, + rd_kafka_ListConsumerGroupsResult_free); + rd_list_add(&rko_partial->rko_u.admin_result.results, result); + + rd_list_destroy(&valid); + rd_list_destroy(&errors); + + return rko_partial; +} + +/** + * @brief Helper to create a mock partial error result op. + */ +static rd_kafka_op_t * +ut_make_ListConsumerGroups_partial_error(rd_kafka_resp_err_t err) { + rd_kafka_op_t *rko_partial; + + rko_partial = rd_kafka_op_new(RD_KAFKA_OP_ADMIN_RESULT); + rko_partial->rko_evtype = RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT; + rko_partial->rko_err = err; + + rd_list_init(&rko_partial->rko_u.admin_result.results, 0, NULL); + + return rko_partial; +} + +/** + * @brief Helper to create a fanout op for merge testing. + */ +static rd_kafka_op_t *ut_make_ListConsumerGroups_fanout(void) { + rd_kafka_op_t *rko_fanout; + + rko_fanout = rd_kafka_op_new(RD_KAFKA_OP_ADMIN_FANOUT); + rd_list_init(&rko_fanout->rko_u.admin_request.fanout.results, 1, + rd_kafka_ListConsumerGroupsResult_free); + rd_list_init(&rko_fanout->rko_u.admin_request.args, 0, NULL); + + return rko_fanout; +} + +/** + * @brief Verify that the merged result contains exactly the expected + * group ids (in any order) and no duplicates. + * + * @returns 0 on success, 1 on failure. + */ +static int ut_verify_ListConsumerGroups_result(const rd_kafka_op_t *rko_fanout, + const char **expected_group_ids, + int expected_cnt, + int expected_error_cnt, + const char *test_label) { + rd_kafka_ListConsumerGroupsResult_t *res; + int valid_cnt, error_cnt, i, j; + + res = rd_list_elem(&rko_fanout->rko_u.admin_request.fanout.results, 0); + RD_UT_ASSERT(res != NULL, "%s: expected non-NULL result", test_label); + + valid_cnt = rd_list_cnt(&res->valid); + error_cnt = rd_list_cnt(&res->errors); + + RD_UT_ASSERT(valid_cnt == expected_cnt, + "%s: expected %d unique groups, got %d", test_label, + expected_cnt, valid_cnt); + + RD_UT_ASSERT(error_cnt == expected_error_cnt, + "%s: expected %d errors, got %d", test_label, + expected_error_cnt, error_cnt); + + /* Verify each expected group is present exactly once */ + for (i = 0; i < expected_cnt; i++) { + int found = 0; + for (j = 0; j < valid_cnt; j++) { + const rd_kafka_ConsumerGroupListing_t *grp = + rd_list_elem(&res->valid, j); + if (!strcmp(grp->group_id, expected_group_ids[i])) + found++; + } + RD_UT_ASSERT(found == 1, + "%s: expected group '%s' exactly once, " + "found %d times", + test_label, expected_group_ids[i], found); + } + + return 0; +} + +/** + * @brief Test ListConsumerGroups_response_merge deduplication. + */ +static int ut_ListConsumerGroups_response_merge(void) { + rd_kafka_op_t *rko_fanout; + rd_kafka_op_t *rko_partial; + + RD_UT_BEGIN(); + + /* + * Test 1: Two brokers return identical groups. + * Merge {A, B} then {A, B} → expect {A, B} + */ + { + const char *groups1[] = {"groupA", "groupB"}; + const char *groups2[] = {"groupA", "groupB"}; + const char *expected[] = {"groupA", "groupB"}; + + rko_fanout = ut_make_ListConsumerGroups_fanout(); + + rko_partial = ut_make_ListConsumerGroups_partial(groups1, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(groups2, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + if (ut_verify_ListConsumerGroups_result( + rko_fanout, expected, 2, 0, + "identical groups from two brokers")) + return 1; + + rd_kafka_op_destroy(rko_fanout); + } + + /* + * Test 2: Two brokers return completely disjoint groups. + * Merge {A, B} then {C, D} → expect {A, B, C, D} + */ + { + const char *groups1[] = {"groupA", "groupB"}; + const char *groups2[] = {"groupC", "groupD"}; + const char *expected[] = {"groupA", "groupB", "groupC", + "groupD"}; + + rko_fanout = ut_make_ListConsumerGroups_fanout(); + + rko_partial = ut_make_ListConsumerGroups_partial(groups1, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(groups2, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + if (ut_verify_ListConsumerGroups_result( + rko_fanout, expected, 4, 0, + "disjoint groups from two brokers")) + return 1; + + rd_kafka_op_destroy(rko_fanout); + } + + /* + * Test 3: Three brokers with complex overlap. + * Broker1: {A, B, C}, Broker2: {B, C, D}, Broker3: {C, D, E} + * → expect {A, B, C, D, E} + */ + { + const char *groups1[] = {"groupA", "groupB", "groupC"}; + const char *groups2[] = {"groupB", "groupC", "groupD"}; + const char *groups3[] = {"groupC", "groupD", "groupE"}; + const char *expected[] = {"groupA", "groupB", "groupC", + "groupD", "groupE"}; + + rko_fanout = ut_make_ListConsumerGroups_fanout(); + + rko_partial = ut_make_ListConsumerGroups_partial(groups1, 3); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(groups2, 3); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(groups3, 3); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + if (ut_verify_ListConsumerGroups_result( + rko_fanout, expected, 5, 0, + "three brokers with complex overlap")) + return 1; + + rd_kafka_op_destroy(rko_fanout); + } + + /* + * Test 4: Duplicates with interleaved error. + * Broker1: {A, B}, Broker2: error, Broker3: {A, C} + * → expect {A, B, C} + 1 error + */ + { + const char *groups1[] = {"groupA", "groupB"}; + const char *groups3[] = {"groupA", "groupC"}; + const char *expected[] = {"groupA", "groupB", "groupC"}; + + rko_fanout = ut_make_ListConsumerGroups_fanout(); + + rko_partial = ut_make_ListConsumerGroups_partial(groups1, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial_error( + RD_KAFKA_RESP_ERR__TIMED_OUT); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(groups3, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + if (ut_verify_ListConsumerGroups_result( + rko_fanout, expected, 3, 1, + "duplicates with interleaved error")) + return 1; + + rd_kafka_op_destroy(rko_fanout); + } + + /* + * Test 5: Empty broker response interleaved. + * Broker1: {A, B}, Broker2: {}, Broker3: {B, C} + * → expect {A, B, C} + */ + { + const char *groups1[] = {"groupB", "groupA"}; + const char *groups3[] = {"groupC", "groupB"}; + const char *expected[] = {"groupA", "groupB", "groupC"}; + + rko_fanout = ut_make_ListConsumerGroups_fanout(); + + rko_partial = ut_make_ListConsumerGroups_partial(groups1, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(NULL, 0); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + rko_partial = ut_make_ListConsumerGroups_partial(groups3, 2); + rd_kafka_ListConsumerGroups_response_merge(rko_fanout, + rko_partial); + rd_kafka_op_destroy(rko_partial); + + if (ut_verify_ListConsumerGroups_result( + rko_fanout, expected, 3, 0, + "empty broker response interleaved")) + return 1; + + rd_kafka_op_destroy(rko_fanout); + } + + RD_UT_PASS(); +} + +int unittest_admin(void) { + int fails = 0; + fails += ut_ListConsumerGroups_response_merge(); + return fails; +} + +/**@}*/ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_admin.h b/lib/librdkafka-2.15.0/src/rdkafka_admin.h similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_admin.h rename to lib/librdkafka-2.15.0/src/rdkafka_admin.h index c84849ea666..8129078bc0b 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_admin.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_admin.h @@ -35,10 +35,7 @@ #include "rdmap.h" #include "rdkafka_error.h" #include "rdkafka_confval.h" - -#if WITH_SSL && OPENSSL_VERSION_NUMBER >= 0x10101000L -#include -#endif +#include "rdrand.h" #if WITH_SSL typedef struct rd_kafka_broker_s rd_kafka_broker_t; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_assignment.c b/lib/librdkafka-2.15.0/src/rdkafka_assignment.c similarity index 80% rename from lib/librdkafka-2.10.1/src/rdkafka_assignment.c rename to lib/librdkafka-2.15.0/src/rdkafka_assignment.c index 6d1f01913f9..2d0120582f0 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_assignment.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_assignment.c @@ -96,6 +96,7 @@ */ #include "rdkafka_int.h" +#include "rdkafka_cgrp.h" #include "rdkafka_offset.h" #include "rdkafka_request.h" @@ -332,6 +333,47 @@ static void rd_kafka_assignment_handle_OffsetFetch(rd_kafka_t *rk, rd_kafka_topic_partition_list_destroy(offsets); } +static void rd_kafka_share_assignment_serve_removals(rd_kafka_t *rk) { + rd_kafka_topic_partition_t *rktpar; + + RD_KAFKA_TPLIST_FOREACH(rktpar, rk->rk_consumer.assignment.removed) { + rd_kafka_toppar_t *rktp = + rd_kafka_topic_partition_ensure_toppar( + rk, rktpar, rd_true); /* Borrow ref */ + + /* Remove from pending list if present */ + rd_kafka_topic_partition_list_del( + rk->rk_consumer.assignment.pending, rktpar->topic, + rktpar->partition); + + /* Detach the partition from the cgrp directly. This sends + * SHARE_SESSION_PARTITION_REMOVE to the partition's broker + * so it queues a forget on the next ShareFetch. */ + rd_dassert(rktp->rktp_cgrp); + rd_kafka_cgrp_partition_del(rktp->rktp_cgrp, rktp); + + /* desired_del requires both topic_wrlock and toppar_lock. */ + rd_kafka_topic_wrlock(rktp->rktp_rkt); + rd_kafka_toppar_lock(rktp); + + /* No longer desired */ + rd_kafka_toppar_desired_del(rktp); + + rd_kafka_toppar_unlock(rktp); + rd_kafka_topic_wrunlock(rktp->rktp_rkt); + + rd_kafka_dbg(rk, CGRP, "REMOVE", + "Removing %s [%" PRId32 "] from share assignment", + rktpar->topic, rktpar->partition); + } + + rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "REMOVE", + "Served %d removed partition(s).", + rk->rk_consumer.assignment.removed->cnt); + + rd_kafka_topic_partition_list_clear(rk->rk_consumer.assignment.removed); +} + /** * @brief Decommission all partitions in the removed list. @@ -434,6 +476,28 @@ static int rd_kafka_assignment_serve_removals(rd_kafka_t *rk) { rk->rk_consumer.wait_commit_cnt; } +static void rd_kafka_share_assignment_serve_pending(rd_kafka_t *rk) { + int i; + + for (i = rk->rk_consumer.assignment.pending->cnt - 1; i >= 0; i--) { + rd_kafka_topic_partition_t *rktpar = + &rk->rk_consumer.assignment.pending->elems[i]; + rd_kafka_toppar_t *rktp = + rd_kafka_topic_partition_ensure_toppar(rk, rktpar, rd_true); + + rd_kafka_dbg(rk, CGRP, "SRVPEND", + "Adding share partition %s [%" PRId32 + "] to assignment", + rktpar->topic, rktpar->partition); + + rktp->rktp_cgrp = rk->rk_cgrp; + + rd_kafka_cgrp_partition_add(rktp->rktp_cgrp, rktp); + rd_kafka_topic_partition_list_del_by_idx( + rk->rk_consumer.assignment.pending, i); + } +} + /** * @brief Serve all partitions in the pending list. @@ -601,7 +665,33 @@ static int rd_kafka_assignment_serve_pending(rd_kafka_t *rk) { rk->rk_consumer.assignment.queried->cnt; } +/** + * @brief Serve updates to the share assignment. + * + * Call on: + * - assignment changes + */ +void rd_kafka_share_assignment_serve(rd_kafka_t *rk) { + + rd_kafka_assignment_dump(rk); + /* Serve any partitions that should be removed */ + if (rk->rk_consumer.assignment.removed->cnt > 0) + rd_kafka_share_assignment_serve_removals(rk); + + /* Serve pending partitions */ + if (rk->rk_consumer.assignment.pending->cnt > 0) + rd_kafka_share_assignment_serve_pending(rk); + + /* Share-consumer serve always completes synchronously: removals + * don't wait for a fetcher stop (no fetcher), pending doesn't wait + * for an OffsetFetch (no offset commits). Signal done immediately. */ + rd_kafka_cgrp_assignment_done(rk->rk_cgrp); + + rd_kafka_dbg(rk, CGRP, "ASSIGNMENT", + "Share assignment served: %d partition(s) assigned", + rk->rk_consumer.assignment.all->cnt); +} /** * @brief Serve updates to the assignment. @@ -612,6 +702,10 @@ static int rd_kafka_assignment_serve_pending(rd_kafka_t *rk) { * - partition fetcher is stopped */ void rd_kafka_assignment_serve(rd_kafka_t *rk) { + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_kafka_share_assignment_serve(rk); + return; + } int inp_removals = 0; int inp_pending = 0; @@ -670,6 +764,32 @@ rd_bool_t rd_kafka_assignment_in_progress(rd_kafka_t *rk) { } +/** + * @brief Share-consumer variant of rd_kafka_assignment_clear(). + */ +static int rd_kafka_share_assignment_clear(rd_kafka_t *rk) { + int cnt = rk->rk_consumer.assignment.all->cnt; + + if (cnt == 0) { + rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "CLEARASSIGN", + "No current share assignment to clear"); + return 0; + } + + rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "CLEARASSIGN", + "Clearing current share assignment of %d partition(s)", + cnt); + + rd_kafka_topic_partition_list_clear(rk->rk_consumer.assignment.pending); + + rd_kafka_topic_partition_list_add_list( + rk->rk_consumer.assignment.removed, rk->rk_consumer.assignment.all); + rd_kafka_topic_partition_list_clear(rk->rk_consumer.assignment.all); + + return cnt; +} + + /** * @brief Clear the current assignment. * @@ -679,7 +799,12 @@ rd_bool_t rd_kafka_assignment_in_progress(rd_kafka_t *rk) { * @returns the number of partitions removed. */ int rd_kafka_assignment_clear(rd_kafka_t *rk) { - int cnt = rk->rk_consumer.assignment.all->cnt; + int cnt; + + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) + return rd_kafka_share_assignment_clear(rk); + + cnt = rk->rk_consumer.assignment.all->cnt; if (cnt == 0) { rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "CLEARASSIGN", @@ -704,6 +829,66 @@ int rd_kafka_assignment_clear(rd_kafka_t *rk) { } +/** + * @brief Share-consumer variant of rd_kafka_assignment_add(). + */ +static rd_kafka_error_t * +rd_kafka_share_assignment_add(rd_kafka_t *rk, + rd_kafka_topic_partition_list_t *partitions) { + rd_bool_t was_empty = rk->rk_consumer.assignment.all->cnt == 0; + int i; + + /* Reject duplicates inside the input list and overlap with the + * current assignment. Offset validation is omitted — broker owns + * the start position for share consumers. */ + rd_kafka_topic_partition_list_sort(partitions, NULL, NULL); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + const rd_kafka_topic_partition_t *prev = + i > 0 ? &partitions->elems[i - 1] : NULL; + + if (prev && !rd_kafka_topic_partition_cmp(rktpar, prev)) + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__INVALID_ARG, + "Duplicate %s [%" PRId32 "] in input list", + rktpar->topic, rktpar->partition); + + if (rd_kafka_topic_partition_list_find( + rk->rk_consumer.assignment.all, rktpar->topic, + rktpar->partition)) + return rd_kafka_error_new(RD_KAFKA_RESP_ERR__CONFLICT, + "%s [%" PRId32 + "] is already part of the " + "current assignment", + rktpar->topic, + rktpar->partition); + + /* Materialize the toppar (creating topic + toppar if + * missing) and cache it in rktpar->_private so the next + * loop can reuse the lookup. */ + rd_kafka_topic_partition_ensure_toppar(rk, rktpar, rd_true); + } + + rd_kafka_topic_partition_list_add_list(rk->rk_consumer.assignment.all, + partitions); + if (!was_empty) + rd_kafka_topic_partition_list_sort( + rk->rk_consumer.assignment.all, NULL, NULL); + + rd_kafka_topic_partition_list_add_list( + rk->rk_consumer.assignment.pending, partitions); + + rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_CGRP, "ASSIGNMENT", + "Added %d partition(s) to share assignment which " + "now consists of %d partition(s), %d in pending state", + partitions->cnt, rk->rk_consumer.assignment.all->cnt, + rk->rk_consumer.assignment.pending->cnt); + + return NULL; +} + + /** * @brief Adds \p partitions to the current assignment. * @@ -719,6 +904,9 @@ rd_kafka_assignment_add(rd_kafka_t *rk, rd_bool_t was_empty = rk->rk_consumer.assignment.all->cnt == 0; int i; + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) + return rd_kafka_share_assignment_add(rk, partitions); + /* Make sure there are no duplicates, invalid partitions, or * invalid offsets in the input partitions. */ rd_kafka_topic_partition_list_sort(partitions, NULL, NULL); @@ -822,6 +1010,71 @@ rd_kafka_assignment_add(rd_kafka_t *rk, } +/** + * @brief Share-consumer variant of rd_kafka_assignment_subtract(). + */ +static rd_kafka_error_t *rd_kafka_share_assignment_subtract( + rd_kafka_t *rk, + rd_kafka_topic_partition_list_t *partitions) { + int i; + int assignment_pre_cnt; + + if (rk->rk_consumer.assignment.all->cnt == 0 && partitions->cnt > 0) + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__INVALID_ARG, + "Can't subtract from empty assignment"); + + rd_kafka_topic_partition_list_sort(partitions, NULL, NULL); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + + if (!rd_kafka_topic_partition_list_find( + rk->rk_consumer.assignment.all, rktpar->topic, + rktpar->partition)) + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__INVALID_ARG, + "%s [%" PRId32 + "] can't be unassigned since " + "it is not in the current assignment", + rktpar->topic, rktpar->partition); + + rd_kafka_topic_partition_ensure_toppar(rk, rktpar, rd_true); + } + + assignment_pre_cnt = rk->rk_consumer.assignment.all->cnt; + + for (i = partitions->cnt - 1; i >= 0; i--) { + const rd_kafka_topic_partition_t *rktpar = + &partitions->elems[i]; + + if (!rd_kafka_topic_partition_list_del( + rk->rk_consumer.assignment.all, rktpar->topic, + rktpar->partition)) + RD_BUG("Removed partition %s [%" PRId32 + "] not found in assignment.all", + rktpar->topic, rktpar->partition); + + rd_kafka_topic_partition_list_del( + rk->rk_consumer.assignment.pending, rktpar->topic, + rktpar->partition); + + rd_kafka_topic_partition_list_add_copy( + rk->rk_consumer.assignment.removed, rktpar); + } + + rd_kafka_dbg(rk, CGRP, "REMOVEASSIGN", + "Removed %d partition(s) from share assignment of " + "%d partition(s)", + partitions->cnt, assignment_pre_cnt); + + if (rk->rk_consumer.assignment.all->cnt == 0) + rd_assert(rk->rk_consumer.assignment.pending->cnt == 0); + + return NULL; +} + + /** * @brief Remove \p partitions from the current assignment. * @@ -838,6 +1091,9 @@ rd_kafka_assignment_subtract(rd_kafka_t *rk, int matched_queried_partitions = 0; int assignment_pre_cnt; + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) + return rd_kafka_share_assignment_subtract(rk, partitions); + if (rk->rk_consumer.assignment.all->cnt == 0 && partitions->cnt > 0) return rd_kafka_error_new( RD_KAFKA_RESP_ERR__INVALID_ARG, @@ -948,6 +1204,14 @@ void rd_kafka_assignment_partition_stopped(rd_kafka_t *rk, */ void rd_kafka_assignment_pause(rd_kafka_t *rk, const char *reason) { + /** + * TODO KIP-932: Check how can remove the invocation + * of this function altogether for share consumers. + */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + return; + } + if (rk->rk_consumer.assignment.all->cnt == 0) return; @@ -966,6 +1230,14 @@ void rd_kafka_assignment_pause(rd_kafka_t *rk, const char *reason) { */ void rd_kafka_assignment_resume(rd_kafka_t *rk, const char *reason) { + /** + * TODO KIP-932: Check how can remove the invocation + * of this function altogether for share consumers. + */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + return; + } + if (rk->rk_consumer.assignment.all->cnt == 0) return; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_assignment.h b/lib/librdkafka-2.15.0/src/rdkafka_assignment.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_assignment.h rename to lib/librdkafka-2.15.0/src/rdkafka_assignment.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_assignor.c b/lib/librdkafka-2.15.0/src/rdkafka_assignor.c similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_assignor.c rename to lib/librdkafka-2.15.0/src/rdkafka_assignor.c index 465568c41da..38e10eb3acc 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_assignor.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_assignor.c @@ -240,6 +240,13 @@ static void rd_kafka_assignor_topic_destroy(rd_kafka_assignor_topic_t *at) { rd_free(at); } +/** + * @brief Destroy-variant suitable for rd_list free_cb use. + */ +static void rd_kafka_assignor_topic_free(void *ptr) { + rd_kafka_assignor_topic_destroy((rd_kafka_assignor_topic_t *)ptr); +} + int rd_kafka_assignor_topic_cmp(const void *_a, const void *_b) { const rd_kafka_assignor_topic_t *a = *(const rd_kafka_assignor_topic_t *const *)_a; @@ -267,7 +274,7 @@ rd_kafka_member_subscriptions_map(rd_kafka_cgrp_t *rkcg, rd_kafka_metadata_get_internal(metadata); rd_list_init(eligible_topics, RD_MIN(metadata->topic_cnt, 10), - (void *)rd_kafka_assignor_topic_destroy); + rd_kafka_assignor_topic_free); /* For each topic in the cluster, scan through the member list * to find matching subscriptions. */ @@ -457,6 +464,13 @@ static void rd_kafka_assignor_destroy(rd_kafka_assignor_t *rkas) { rd_free(rkas); } +/** + * @brief Destroy-variant suitable for rd_list free_cb use. + */ +static void rd_kafka_assignor_free(void *ptr) { + rd_kafka_assignor_destroy((rd_kafka_assignor_t *)ptr); +} + /** * @brief Check that the rebalance protocol of all enabled assignors is @@ -580,7 +594,7 @@ int rd_kafka_assignors_init(rd_kafka_t *rk, char *errstr, size_t errstr_size) { int idx = 0; rd_list_init(&rk->rk_conf.partition_assignors, 3, - (void *)rd_kafka_assignor_destroy); + rd_kafka_assignor_free); /* Initialize builtin assignors (ignore errors) */ rd_kafka_range_assignor_init(rk); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_assignor.h b/lib/librdkafka-2.15.0/src/rdkafka_assignor.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_assignor.h rename to lib/librdkafka-2.15.0/src/rdkafka_assignor.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_aux.c b/lib/librdkafka-2.15.0/src/rdkafka_aux.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_aux.c rename to lib/librdkafka-2.15.0/src/rdkafka_aux.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_aux.h b/lib/librdkafka-2.15.0/src/rdkafka_aux.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_aux.h rename to lib/librdkafka-2.15.0/src/rdkafka_aux.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_background.c b/lib/librdkafka-2.15.0/src/rdkafka_background.c similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_background.c rename to lib/librdkafka-2.15.0/src/rdkafka_background.c index a9c96606c0d..47b2d30ba4e 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_background.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_background.c @@ -111,6 +111,7 @@ int rd_kafka_background_thread_main(void *arg) { rd_kafka_set_thread_name("background"); rd_kafka_set_thread_sysname("rdk:bg"); + rd_kafka_thread_srand(rk, rd_true /* we're in an internal thread */); rd_kafka_interceptors_on_thread_start(rk, RD_KAFKA_THREAD_BACKGROUND); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_broker.c b/lib/librdkafka-2.15.0/src/rdkafka_broker.c similarity index 89% rename from lib/librdkafka-2.10.1/src/rdkafka_broker.c rename to lib/librdkafka-2.15.0/src/rdkafka_broker.c index 48a67a5cb0a..d8e5c48e819 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_broker.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_broker.c @@ -68,6 +68,7 @@ #include "rdkafka_idempotence.h" #include "rdkafka_txnmgr.h" #include "rdkafka_fetcher.h" +#include "rdkafka_share_acknowledgement.h" #include "rdtime.h" #include "rdcrc32.h" #include "rdrand.h" @@ -310,6 +311,32 @@ int16_t rd_kafka_broker_ApiVersion_supported(rd_kafka_broker_t *rkb, rkb, ApiKey, minver, maxver, featuresp, rd_true /* do_lock */); } +/** + * @brief Check that at least a ApiVersion greater or equal to + * \p minver exists for \p ApiKey. + * + * @returns `rd_true` if the broker supports \p ApiKey with + * a version greater than or equal to \p minver, else `rd_false`. + * @locks none + * @locks_acquired rd_kafka_broker_lock() + * @locality any + */ +rd_bool_t rd_kafka_broker_ApiVersion_at_least(rd_kafka_broker_t *rkb, + int16_t ApiKey, + int16_t minver) { + return rd_kafka_broker_ApiVersion_supported0( + rkb, ApiKey, minver, RD_KAFKAP_RPC_VERSION_MAX, NULL, + rd_true /* do_lock */) != -1; +} + +rd_bool_t rd_kafka_broker_ApiVersion_at_least_no_lock(rd_kafka_broker_t *rkb, + int16_t ApiKey, + int16_t minver) { + return rd_kafka_broker_ApiVersion_supported0( + rkb, ApiKey, minver, RD_KAFKAP_RPC_VERSION_MAX, NULL, + rd_false /* don't lock rkb */) != -1; +} + /** * @brief Set broker state. * @@ -320,7 +347,9 @@ int16_t rd_kafka_broker_ApiVersion_supported(rd_kafka_broker_t *rkb, * @locality broker thread */ void rd_kafka_broker_set_state(rd_kafka_broker_t *rkb, int state) { - rd_bool_t trigger_monitors = rd_false; + rd_bool_t trigger_monitors = rd_false, + reset_any_broker_down_reported = rd_false, + skip_broker_down = rkb->rkb_c.skip_broker_down; if ((int)rkb->rkb_state == state) return; @@ -330,15 +359,20 @@ void rd_kafka_broker_set_state(rd_kafka_broker_t *rkb, int state) { rd_kafka_broker_state_names[rkb->rkb_state], rd_kafka_broker_state_names[state]); + if (rd_kafka_broker_state_is_down(state) && skip_broker_down) + /* Reset the flag so if it disconnects again it won't be + * treated as a planned disconnection. */ + rkb->rkb_c.skip_broker_down = rd_false; + if (rkb->rkb_source == RD_KAFKA_INTERNAL || RD_KAFKA_BROKER_IS_LOGICAL(rkb)) { /* no-op */ - } else if (rd_kafka_broker_state_is_down(state) && - !rkb->rkb_down_reported) { + } else if (rd_kafka_broker_state_is_down(state) && !skip_broker_down && + /* Only 0 -> 1 */ + rd_atomic32_set(&rkb->rkb_down_reported, 1) == 0) { + /* Propagate ALL_BROKERS_DOWN event if all brokers are - * now down, unless we're terminating. - * Only trigger for brokers that has an address set, - * e.g., not logical brokers that lost their address. */ + * now down, unless we're terminating. */ if (rd_atomic32_add(&rkb->rkb_rk->rk_broker_down_cnt, 1) == rd_atomic32_get(&rkb->rkb_rk->rk_broker_cnt) - rd_atomic32_get( @@ -353,27 +387,37 @@ void rd_kafka_broker_set_state(rd_kafka_broker_t *rkb, int state) { rd_atomic32_get( &rkb->rkb_rk->rk_logical_broker_cnt)); } - rkb->rkb_down_reported = 1; } else if (rd_kafka_broker_state_is_up(state) && - rkb->rkb_down_reported) { + /* Only 1 -> 0 */ + rd_atomic32_set(&rkb->rkb_down_reported, 0) == 1) { rd_atomic32_sub(&rkb->rkb_rk->rk_broker_down_cnt, 1); - rkb->rkb_down_reported = 0; } if (rkb->rkb_source != RD_KAFKA_INTERNAL) { if (rd_kafka_broker_state_is_up(state) && !rd_kafka_broker_state_is_up(rkb->rkb_state)) { - /* Up -> Down */ - if (!RD_KAFKA_BROKER_IS_LOGICAL(rkb)) + /* ~Up -> Up */ + if (!RD_KAFKA_BROKER_IS_LOGICAL(rkb)) { rd_atomic32_add(&rkb->rkb_rk->rk_broker_up_cnt, 1); + /* If at least one broker connects we reset + * the down counter to try again with rest of + * brokers. Otherwise, a single broker + * connection can cause `ALL_BROKERS_DOWN` to + * get triggered even if the connection to rest + * of brokers was restored but they didn't + * attempt a re-connection because of sparse + * broker connections. */ + reset_any_broker_down_reported = rd_true; + } + trigger_monitors = rd_true; } else if (rd_kafka_broker_state_is_up(rkb->rkb_state) && !rd_kafka_broker_state_is_up(state)) { - /* ~Down(!Up) -> Up */ + /* Up -> Down */ if (!RD_KAFKA_BROKER_IS_LOGICAL(rkb)) rd_atomic32_sub(&rkb->rkb_rk->rk_broker_up_cnt, 1); @@ -394,6 +438,14 @@ void rd_kafka_broker_set_state(rd_kafka_broker_t *rkb, int state) { rkb->rkb_state = state; rkb->rkb_ts_state = rd_clock(); + if (reset_any_broker_down_reported) { + rd_kafka_broker_unlock(rkb); + /* Releases rkb->rkb_lock to respect + * lock ordering and avoid deadlocks */ + rd_kafka_reset_any_broker_down_reported(rkb->rkb_rk); + rd_kafka_broker_lock(rkb); + } + if (trigger_monitors) rd_kafka_broker_trigger_monitors(rkb); @@ -453,34 +505,31 @@ static void rd_kafka_broker_set_error(rd_kafka_broker_t *rkb, /* Provide more meaningful error messages in certain cases */ if (err == RD_KAFKA_RESP_ERR__TRANSPORT && - !strcmp(errstr, "Disconnected")) { + rd_kafka_transport_error_disconnected(fmt)) { if (rkb->rkb_state == RD_KAFKA_BROKER_STATE_APIVERSION_QUERY) { /* A disconnect while requesting ApiVersion typically * means we're connecting to a SSL-listener as * PLAINTEXT, but may also be caused by connecting to * a broker that does not support ApiVersion (<0.10). */ - if (rkb->rkb_proto != RD_KAFKA_PROTO_SSL && - rkb->rkb_proto != RD_KAFKA_PROTO_SASL_SSL) - rd_kafka_broker_set_error( - rkb, level, err, - "Disconnected while requesting " + rkb->rkb_proto != RD_KAFKA_PROTO_SASL_SSL) { + ofe = rd_snprintf( + errstr + of, sizeof(errstr) - of, "%s", + ": requesting " "ApiVersion: " "might be caused by incorrect " "security.protocol configuration " "(connecting to a SSL listener?) or " "broker version is < 0.10 " - "(see api.version.request)", - ap /*ignored*/); - else - rd_kafka_broker_set_error( - rkb, level, err, - "Disconnected while requesting " + "(see api.version.request)"); + } else { + ofe = rd_snprintf( + errstr + of, sizeof(errstr) - of, "%s", + ": requesting " "ApiVersion: " "might be caused by broker version " - "< 0.10 (see api.version.request)", - ap /*ignored*/); - return; + "< 0.10 (see api.version.request)"); + } } else if (rkb->rkb_state == RD_KAFKA_BROKER_STATE_UP && state_duration_ms < 2000 /*2s*/ && @@ -488,18 +537,20 @@ static void rd_kafka_broker_set_error(rd_kafka_broker_t *rkb, RD_KAFKA_PROTO_SASL_SSL && rkb->rkb_rk->rk_conf.security_protocol != RD_KAFKA_PROTO_SASL_PLAINTEXT) { + /* If disconnected shortly after transitioning to UP * state it typically means the broker listener is * configured for SASL authentication but the client * is not. */ - rd_kafka_broker_set_error( - rkb, level, err, - "Disconnected: verify that security.protocol " - "is correctly configured, broker might " - "require SASL authentication", - ap /*ignored*/); - return; + ofe = + rd_snprintf(errstr + of, sizeof(errstr) - of, "%s", + ": verify that security.protocol " + "is correctly configured, broker might " + "require SASL authentication"); } + if (ofe > sizeof(errstr) - of) + ofe = sizeof(errstr) - of; + of += ofe; } /* Check if error is identical to last error (prior to appending @@ -551,13 +602,14 @@ static void rd_kafka_broker_set_error(rd_kafka_broker_t *rkb, identical ? ": identical to last error" : "", suppress ? ": error log suppressed" : ""); - if (level != LOG_DEBUG && (level <= LOG_CRIT || !suppress)) { + if (level <= LOG_CRIT || (level <= LOG_INFO && !suppress)) { rd_kafka_log(rkb->rkb_rk, level, "FAIL", "%s: %s", rkb->rkb_name, errstr); - /* Send ERR op to application for processing. */ - rd_kafka_q_op_err(rkb->rkb_rk->rk_rep, err, "%s: %s", - rkb->rkb_name, errstr); + if (level <= LOG_ERR) + /* Send ERR op to application for processing. */ + rd_kafka_q_op_err(rkb->rkb_rk->rk_rep, err, "%s: %s", + rkb->rkb_name, errstr); } } @@ -687,6 +739,17 @@ void rd_kafka_broker_fail(rd_kafka_broker_t *rkb, } rd_kafka_toppar_unlock(rktp); + if (RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk)) { + /* Share consumer does not do fetch-from-follower + * (KIP-392), so rktp_broker always equals the + * leader and the FFF unwind compare is a no-op. */ + if (rd_kafka_broker_termination_in_progress(rkb)) { + rd_kafka_toppar_undelegate(rktp); + rd_kafka_toppar_forget_leader(rktp); + } + continue; + } + if (rktp->rktp_leader_id != rktp->rktp_broker_id) { rd_kafka_toppar_delegate_to_leader(rktp); } else if (rd_kafka_broker_termination_in_progress(rkb)) { @@ -721,6 +784,11 @@ void rd_kafka_broker_fail(rd_kafka_broker_t *rkb, } +#define rd_kafka_broker_planned_fail(RKB, ERR, FMT, ...) \ + do { \ + (RKB)->rkb_c.skip_broker_down = rd_true; \ + rd_kafka_broker_fail(RKB, LOG_DEBUG, ERR, FMT, __VA_ARGS__); \ + } while (0) /** * @brief Handle broker connection close. @@ -1004,11 +1072,11 @@ static void rd_kafka_broker_timeout_scan(rd_kafka_broker_t *rkb, rd_ts_t now) { 1000.0f)); else rttinfo[0] = 0; - rd_kafka_broker_fail(rkb, LOG_ERR, - RD_KAFKA_RESP_ERR__TIMED_OUT, - "%i request(s) timed out: " - "disconnect%s", - rkb->rkb_req_timeouts, rttinfo); + rd_kafka_broker_planned_fail( + rkb, RD_KAFKA_RESP_ERR__TIMED_OUT, + "%i request(s) timed out: " + "disconnect%s", + rkb->rkb_req_timeouts, rttinfo); } } } @@ -1047,8 +1115,9 @@ static int rd_kafka_broker_resolve(rd_kafka_broker_t *rkb, int save_idx = 0; if (!*nodename && rkb->rkb_source == RD_KAFKA_LOGICAL) { - rd_kafka_broker_fail(rkb, LOG_DEBUG, RD_KAFKA_RESP_ERR__RESOLVE, - "Logical broker has no address yet"); + rd_kafka_broker_planned_fail( + rkb, RD_KAFKA_RESP_ERR__RESOLVE, "%s", + "Logical broker has no address yet"); return -1; } @@ -1161,10 +1230,7 @@ static void rd_kafka_buf_finalize(rd_kafka_t *rk, rd_kafka_buf_t *rkbuf) { rd_assert(!(rkbuf->rkbuf_flags & RD_KAFKA_OP_F_NEED_MAKE)); - if (rkbuf->rkbuf_flags & RD_KAFKA_OP_F_FLEXVER) { - /* Empty struct tags */ - rd_kafka_buf_write_i8(rkbuf, 0); - } + rd_kafka_buf_write_tags_empty(rkbuf); /* Calculate total request buffer length. */ totsize = rd_buf_len(&rkbuf->rkbuf_buf) - 4; @@ -2111,7 +2177,7 @@ int rd_kafka_recv(rd_kafka_broker_t *rkb) { err_parse: err = rkbuf->rkbuf_err; err: - if (!strcmp(errstr, "Disconnected")) + if (rd_kafka_transport_error_disconnected(errstr)) rd_kafka_broker_conn_closed(rkb, err, errstr); else rd_kafka_broker_fail(rkb, LOG_ERR, err, "Receive failed: %s", @@ -2214,53 +2280,6 @@ rd_kafka_broker_reconnect_backoff(const rd_kafka_broker_t *rkb, rd_ts_t now) { return (int)(remains / 1000); } - -/** - * @brief Unittest for reconnect.backoff.ms - */ -static int rd_ut_reconnect_backoff(void) { - rd_kafka_broker_t rkb = RD_ZERO_INIT; - rd_kafka_conf_t conf = {.reconnect_backoff_ms = 10, - .reconnect_backoff_max_ms = 90}; - rd_ts_t now = 1000000; - int backoff; - - rkb.rkb_reconnect_backoff_ms = conf.reconnect_backoff_ms; - - /* broker's backoff is the initial reconnect.backoff.ms=10 */ - rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); - backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); - RD_UT_ASSERT_RANGE(backoff, 7, 15, "%d"); - - /* .. 20 */ - rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); - backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); - RD_UT_ASSERT_RANGE(backoff, 15, 30, "%d"); - - /* .. 40 */ - rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); - backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); - RD_UT_ASSERT_RANGE(backoff, 30, 60, "%d"); - - /* .. 80, the jitter is capped at reconnect.backoff.max.ms=90 */ - rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); - backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); - RD_UT_ASSERT_RANGE(backoff, 60, conf.reconnect_backoff_max_ms, "%d"); - - /* .. 90, capped by reconnect.backoff.max.ms */ - rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); - backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); - RD_UT_ASSERT_RANGE(backoff, 67, conf.reconnect_backoff_max_ms, "%d"); - - /* .. 90, should remain at capped value. */ - rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); - backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); - RD_UT_ASSERT_RANGE(backoff, 67, conf.reconnect_backoff_max_ms, "%d"); - - RD_UT_PASS(); -} - - /** * @brief Initiate asynchronous connection attempt to the next address * in the broker's address list. @@ -2956,10 +2975,18 @@ void rd_kafka_broker_buf_retry(rd_kafka_broker_t *rkb, rd_kafka_buf_t *rkbuf) { rd_atomic64_add(&rkb->rkb_c.tx_retries, 1); /* In some cases, failed Produce requests do not increment the retry * count, see rd_kafka_handle_Produce_error. */ - if (rkbuf->rkbuf_retries > 0) - backoff = (1 << (rkbuf->rkbuf_retries - 1)) * + if (rkbuf->rkbuf_retries > 0) { + int shift = rkbuf->rkbuf_retries - 1; + + /* Cap shift at 34 to prevent overflow in final calculation. + * Accounts for multiplication by retry_backoff_ms (max 300000) + * and jitter (max 1200). */ + if (shift > 34) + shift = 34; + + backoff = ((int64_t)1 << shift) * (rkb->rkb_rk->rk_conf.retry_backoff_ms); - else + } else backoff = rkb->rkb_rk->rk_conf.retry_backoff_ms; /* We are multiplying by 10 as (backoff_ms * percent * 1000)/100 -> @@ -3160,6 +3187,156 @@ static rd_kafka_resp_err_t rd_kafka_broker_destroy_error(rd_kafka_t *rk) { : RD_KAFKA_RESP_ERR__DESTROY_BROKER; } + +/** + * @brief Add description. + * + * @locality broker thread + * @locks toppar lock + * @locks broker lock + */ +static void +rd_kafka_broker_share_session_add_remove_toppar(rd_list_t **toppars_add_list, + rd_list_t **toppars_remove_list, + rd_kafka_toppar_t *rktp) { + if (!*toppars_add_list) { + *toppars_add_list = + rd_list_new(1, rd_kafka_toppar_destroy_free); + } + + if (!rd_list_find(*toppars_add_list, rktp, rd_list_cmp_ptr)) + rd_list_add(*toppars_add_list, rd_kafka_toppar_keep(rktp)); + + /* Remove from removing toppars if present there. */ + if (*toppars_remove_list) { + rd_kafka_toppar_t *removed_rktp = + rd_list_remove(*toppars_remove_list, rktp); + if (removed_rktp) { + rd_kafka_toppar_destroy(removed_rktp); + if (rd_list_empty(*toppars_remove_list)) { + rd_list_destroy(*toppars_remove_list); + *toppars_remove_list = NULL; + } + } + } +} + +/** + * @brief Add description. + * + * In some scenarios, we don't have leader information present while assignment + * is done. In which case, when the leader is known later, we need to add the + * toppar to the broker's share fetch session. Being called from two places: 1) + * when a toppar is being added to the assignment in cgrp. 2) when a toppar is + * being added to the leader + * + * @locality broker thread + * @locks toppar lock + * @locks broker lock + */ +static void rd_kafka_broker_share_session_toppar_add(rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp) { + /** + * TODO KIP-932: + * * Check if rktp is present in current session already or not? + * * Check if rktp is already present in toppars_to_add? + */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + if (rkb->rkb_share_fetch_session.epoch < 0) { + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Not adding %.*s [%" PRId32 + "] to share fetch session " + "as share session is already closed", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition); + + return; + } + + rd_kafka_broker_share_session_add_remove_toppar( + &rkb->rkb_share_fetch_session.toppars_to_add, + &rkb->rkb_share_fetch_session.toppars_to_forget, rktp); + rd_rkb_dbg( + rkb, FETCH | RD_KAFKA_DBG_CGRP, "SHARESESSION", + "%s [%" PRId32 "]: queued for ADD on broker (epoch=%" PRId32 + ", toppars_to_add=%d, toppars_to_forget=%d, " + "toppars_in_session=%d)", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + rkb->rkb_share_fetch_session.epoch, + rkb->rkb_share_fetch_session.toppars_to_add + ? rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_to_add) + : 0, + rkb->rkb_share_fetch_session.toppars_to_forget + ? rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_to_forget) + : 0, + rkb->rkb_share_fetch_session.toppars_in_session + ? rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_in_session) + : 0); + } +} + +/** + * @brief Add description. + * + * In some scenarios, we have to move the toppar out of the broker's share fetch + * session like leader migration to another broker. Being called from two + * places: 1) when a toppar is being removed from the assignment in cgrp. 2) + * when a toppar is being removed from the leader. + * + * @locality broker thread + * @locks toppar lock + * @locks broker lock + */ +static void +rd_kafka_broker_share_session_toppar_remove(rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp) { + /** + * TODO KIP-932: + * * Check if rktp is present in current session already or not? No + * need to add if it is not present? + * * Check if rktp is already present in toppars_to_forget? + */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + if (rkb->rkb_share_fetch_session.epoch < 0) { + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Not removing %.*s [%" PRId32 + "] from share fetch session " + "as share session is already closed", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition); + + return; + } + + rd_kafka_broker_share_session_add_remove_toppar( + &rkb->rkb_share_fetch_session.toppars_to_forget, + &rkb->rkb_share_fetch_session.toppars_to_add, rktp); + rd_rkb_dbg( + rkb, FETCH | RD_KAFKA_DBG_CGRP, "SHARESESSION", + "%s [%" PRId32 + "]: queued for FORGET on broker (epoch=%" PRId32 + ", toppars_to_add=%d, toppars_to_forget=%d, " + "toppars_in_session=%d)", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + rkb->rkb_share_fetch_session.epoch, + rkb->rkb_share_fetch_session.toppars_to_add + ? rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_to_add) + : 0, + rkb->rkb_share_fetch_session.toppars_to_forget + ? rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_to_forget) + : 0, + rkb->rkb_share_fetch_session.toppars_in_session + ? rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_in_session) + : 0); + } +} + /** * @brief Serve a broker op (an op posted by another thread to be handled by * this broker's thread). @@ -3212,9 +3389,9 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { rd_kafka_wrunlock(rkb->rkb_rk); if (updated) { - rd_kafka_broker_fail(rkb, LOG_DEBUG, - RD_KAFKA_RESP_ERR__TRANSPORT, - "Broker hostname updated"); + rd_kafka_broker_planned_fail( + rkb, RD_KAFKA_RESP_ERR__TRANSPORT, "%s", + "Broker hostname updated"); } rd_kafka_brokers_broadcast_state_change(rkb->rkb_rk); @@ -3305,6 +3482,21 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { rd_kafka_broker_lock(rkb); TAILQ_INSERT_TAIL(&rkb->rkb_toppars, rktp, rktp_rkblink); rkb->rkb_toppar_cnt++; + /* TODO KIP-932: Check if we can use + * rkb->rkb_source == RD_KAFKA_LEARNED instead of != + * RD_KAFKA_INTERNAL. + */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk) && + rd_kafka_toppar_is_on_cgrp(rktp, rd_false) && + rkb->rkb_source != RD_KAFKA_INTERNAL) { + rd_rkb_dbg(rkb, FETCH | RD_KAFKA_DBG_CGRP, + "SHARESESSION", + "%s [%" PRId32 + "]: share session JOIN via PARTITION_JOIN", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition); + rd_kafka_broker_share_session_toppar_add(rkb, rktp); + } rd_kafka_broker_unlock(rkb); rktp->rktp_broker = rkb; rd_assert(!rktp->rktp_msgq_wakeup_q); @@ -3324,7 +3516,8 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { "finish before producing to " "new leader"); } - } else if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER) { + } else if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER && + !RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk)) { rktp->rktp_ts_fetch_backoff = 0; } @@ -3366,8 +3559,11 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { } rd_kafka_toppar_unlock(rktp); - /* Remove from fetcher list */ - rd_kafka_toppar_fetch_decide(rktp, rkb, 1 /*force remove*/); + if (!RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + /* Remove from fetcher list */ + rd_kafka_toppar_fetch_decide(rktp, rkb, + 1 /*force remove*/); + } if (rkb->rkb_rk->rk_type == RD_KAFKA_PRODUCER) { /* Purge any ProduceRequests for this toppar @@ -3403,6 +3599,22 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { rd_kafka_broker_lock(rkb); TAILQ_REMOVE(&rkb->rkb_toppars, rktp, rktp_rkblink); rkb->rkb_toppar_cnt--; + /* TODO KIP-932: Check if we can use + * rkb->rkb_source == RD_KAFKA_LEARNED instead of != + * RD_KAFKA_INTERNAL. + */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk) && + rkb->rkb_source != RD_KAFKA_INTERNAL) { + rd_rkb_dbg(rkb, FETCH | RD_KAFKA_DBG_CGRP, + "SHARESESSION", + "%s [%" PRId32 + "]: share session LEAVE via " + "PARTITION_LEAVE", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition); + rd_kafka_broker_share_session_toppar_remove(rkb, rktp); + } + rd_kafka_broker_unlock(rkb); rd_kafka_broker_destroy(rktp->rktp_broker); if (rktp->rktp_msgq_wakeup_q) { @@ -3449,6 +3661,52 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { rd_kafka_brokers_broadcast_state_change(rkb->rkb_rk); break; + case RD_KAFKA_OP_SHARE_FETCH: + rd_rkb_dbg(rkb, BROKER, "SHAREFETCH", + "Received SHARE_FETCH op for broker %s with " + "should_fetch = %d, should_leave = %d", + rd_kafka_broker_name(rkb), + rko->rko_u.share_fetch.should_fetch, + rko->rko_u.share_fetch.should_leave); + + if (rd_kafka_broker_or_instance_terminating(rkb)) { + rd_kafka_dbg(rkb->rkb_rk, BROKER, "SHAREFETCH", + "Ignoring SHARE_FETCH op: " + "instance or broker is terminating"); + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rko, rd_kafka_broker_destroy_error(rkb->rkb_rk)); + } else if (rkb->rkb_state != RD_KAFKA_BROKER_STATE_UP && + !rkb->rkb_reauth_in_progress) { + /* Broker not usable for a share RPC yet. + * + * Exception: during SASL reauthentication the + * connection and share session stay live, so the op is + * let through (it waits in the output queue until + * reauth completes) instead of being bounced, to avoid + * failing acks for records already acquired on the + * broker, which would otherwise be redelivered when + * their lock lapses. + * + * __TRANSPORT signals the caller that the broker is + * transiently unwritable and the op may be retried. */ + rd_kafka_dbg( + rkb->rkb_rk, BROKER, "SHAREFETCH", + "Ignoring SHARE_FETCH op: " + "broker not up (state %s)", + rd_kafka_broker_state_names[rkb->rkb_state]); + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rko, RD_KAFKA_RESP_ERR__TRANSPORT); + } else if (rko->rko_u.share_fetch.should_leave) { + rd_kafka_broker_share_fetch_session_leave(rkb, rko, + rd_clock()); + } else { + rd_kafka_broker_share_rpc(rkb, rko, rd_clock()); + } + + rko = NULL; /* the rko is reused for the reply */ + + break; + case RD_KAFKA_OP_TERMINATE: /* nop: just a wake-up. */ rd_rkb_dbg(rkb, BROKER, "TERM", @@ -3509,9 +3767,8 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { rd_kafka_broker_unlock(rkb); if (do_disconnect) - rd_kafka_broker_fail( - rkb, LOG_DEBUG, - RD_KAFKA_RESP_ERR__TRANSPORT, + rd_kafka_broker_planned_fail( + rkb, RD_KAFKA_RESP_ERR__TRANSPORT, "%s", "Closing connection due to " "nodename change"); } @@ -3536,6 +3793,56 @@ rd_kafka_broker_op_serve(rd_kafka_broker_t *rkb, rd_kafka_op_t *rko) { wakeup = rd_true; break; + case RD_KAFKA_OP_SHARE_SESSION_PARTITION_ADD: + rd_rkb_dbg(rkb, CGRP, "SHARESESSION", + "Received SHARE_SESSION_PARTITION_ADD op for " + "topic %s [%" PRId32 "] (epoch=%" PRId32 ")%s", + rko->rko_rktp->rktp_rkt->rkt_topic->str, + rko->rko_rktp->rktp_partition, + rkb->rkb_share_fetch_session.epoch, + rkb->rkb_source == RD_KAFKA_INTERNAL + ? ", skipping for internal broker" + : ""); + + /* TODO KIP-932: + * 1. Skip this from caller itself + * 2. Check if we can use + * rkb->rkb_source == RD_KAFKA_LEARNED instead of != + * RD_KAFKA_INTERNAL. + */ + if (rkb->rkb_source != RD_KAFKA_INTERNAL) { + rd_kafka_broker_share_session_toppar_add(rkb, + rko->rko_rktp); + } + + break; + + case RD_KAFKA_OP_SHARE_SESSION_PARTITION_REMOVE: + rd_rkb_dbg(rkb, CGRP, "SHARESESSION", + "Received SHARE_SESSION_PARTITION_REMOVE op for " + "topic %s [%" PRId32 "] (epoch=%" PRId32 ")%s", + rko->rko_rktp->rktp_rkt->rkt_topic->str, + rko->rko_rktp->rktp_partition, + rkb->rkb_share_fetch_session.epoch, + rkb->rkb_source == RD_KAFKA_INTERNAL + ? ", skipping for internal broker" + : ""); + + /* TODO KIP-932: + * 1. Skip this from caller itself + * 2. Check if we can use + * rkb->rkb_source == RD_KAFKA_LEARNED instead of != + * RD_KAFKA_INTERNAL. + */ + if (rkb->rkb_source != RD_KAFKA_INTERNAL) + rd_kafka_broker_share_session_toppar_remove( + rkb, rko->rko_rktp); + break; + case RD_KAFKA_OP_SHARE_SESSION_CLEAR: + rd_rkb_dbg(rkb, CGRP, "TERM", + "Received SHARE_SESSION_CLEAR op"); + rd_kafka_broker_share_fetch_session_clear(rkb); + break; default: rd_kafka_assert(rkb->rkb_rk, !*"unhandled op type"); break; @@ -3779,7 +4086,10 @@ static void rd_kafka_broker_internal_serve(rd_kafka_broker_t *rkb, if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER) { /* Consumer */ do { - rd_kafka_broker_consumer_toppars_serve(rkb); + + if (!RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk)) { + rd_kafka_broker_consumer_toppars_serve(rkb); + } wakeup = rd_kafka_broker_ops_io_serve(rkb, abs_timeout); @@ -4254,7 +4564,54 @@ static void rd_kafka_broker_producer_serve(rd_kafka_broker_t *rkb, rd_kafka_broker_unlock(rkb); } +/** + * @brief Share consumer serving + */ +static void rd_kafka_broker_share_consumer_serve(rd_kafka_broker_t *rkb, + rd_ts_t abs_timeout) { + unsigned int initial_state = rkb->rkb_state; + rd_ts_t now; + + rd_kafka_assert(rkb->rkb_rk, thrd_is_current(rkb->rkb_thread)); + + rd_kafka_broker_lock(rkb); + while (!rd_kafka_broker_terminating(rkb) && + rkb->rkb_state == initial_state && + abs_timeout > (now = rd_clock())) { + rd_ts_t min_backoff = abs_timeout; + + rd_kafka_broker_unlock(rkb); + + /* Drives reconnection: requests a persistent connection so a + * down broker reconnects on its own (fetches are op-driven and + * ops only target UP brokers). */ + if (rkb->rkb_toppar_cnt > 0 && + rkb->rkb_share_fetch_session.epoch >= 0 && + rkb->rkb_state != RD_KAFKA_BROKER_STATE_UP) { + /* There are partitions to fetch but the + * connection is not up. */ + rkb->rkb_persistconn.internal++; + } + + /* Move due retry buffers back to the output queue and shorten + * the wakeup to the next pending retry. Used by + * connection-setup and metadata requests (ApiVersion, SASL, + * Metadata), not by ShareFetch/ShareAcknowledge. */ + if (unlikely(rd_atomic32_get(&rkb->rkb_retrybufs.rkbq_cnt) > 0)) + rd_kafka_broker_retry_bufs_move(rkb, &min_backoff); + + if (min_backoff > abs_timeout) + min_backoff = abs_timeout; + + if (rd_kafka_broker_ops_io_serve(rkb, min_backoff)) + return; /* Wakeup */ + + rd_kafka_broker_lock(rkb); + } + + rd_kafka_broker_unlock(rkb); +} /** * Consumer serving @@ -4338,6 +4695,8 @@ static RD_INLINE void rd_kafka_broker_idle_check(rd_kafka_broker_t *rkb) { rd_ts_t ts_recv = rd_atomic64_get(&rkb->rkb_c.ts_recv); rd_ts_t ts_last_activity = RD_MAX(ts_send, ts_recv); int idle_ms; + const int max_jitter_ms = 2000; + rd_dassert(rkb->rkb_rk->rk_conf.connections_max_idle_ms > 0); /* If nothing has been sent yet, use the connection time as * last activity. */ @@ -4346,13 +4705,22 @@ static RD_INLINE void rd_kafka_broker_idle_check(rd_kafka_broker_t *rkb) { idle_ms = (int)((rd_clock() - ts_last_activity) / 1000); - if (likely(idle_ms < rkb->rkb_rk->rk_conf.connections_max_idle_ms)) + if (unlikely(rkb->rkb_c.connections_max_idle_ms == -1)) { + /* Add a different jitter for each broker. */ + rkb->rkb_c.connections_max_idle_ms = + rkb->rkb_rk->rk_conf.connections_max_idle_ms; + if (rkb->rkb_c.connections_max_idle_ms >= 2 * max_jitter_ms) + rkb->rkb_c.connections_max_idle_ms -= + rd_jitter(0, max_jitter_ms); + } + + if (likely(idle_ms < rkb->rkb_c.connections_max_idle_ms)) return; - rd_kafka_broker_fail(rkb, LOG_DEBUG, RD_KAFKA_RESP_ERR__TRANSPORT, - "Connection max idle time exceeded " - "(%dms since last activity)", - idle_ms); + rd_kafka_broker_planned_fail(rkb, RD_KAFKA_RESP_ERR__TRANSPORT, + "Connection max idle time exceeded " + "(%dms since last activity)", + idle_ms); } @@ -4454,6 +4822,8 @@ static void rd_kafka_broker_serve(rd_kafka_broker_t *rkb, int timeout_ms) { if (rkb->rkb_rk->rk_type == RD_KAFKA_PRODUCER) rd_kafka_broker_producer_serve(rkb, abs_timeout); + else if (RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk)) + rd_kafka_broker_share_consumer_serve(rkb, abs_timeout); else if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER) rd_kafka_broker_consumer_serve(rkb, abs_timeout); @@ -4484,6 +4854,7 @@ static int rd_kafka_broker_thread_main(void *arg) { rd_kafka_set_thread_name("%s", rkb->rkb_name); rd_kafka_set_thread_sysname("rdk:broker%" PRId32, rkb->rkb_nodeid); + rd_kafka_thread_srand(rk, rd_true /* we're in an internal thread */); rd_kafka_interceptors_on_thread_start(rk, RD_KAFKA_THREAD_BROKER); @@ -4720,7 +5091,8 @@ static int rd_kafka_broker_thread_main(void *arg) { if (RD_KAFKA_BROKER_IS_LOGICAL(rkb)) { rd_atomic32_sub(&rkb->rkb_rk->rk_logical_broker_cnt, 1); - } else if (rkb->rkb_down_reported) { + } else if (rd_atomic32_set(&rkb->rkb_down_reported, 0) == 1) { + /* Only 1 -> 0 */ rd_atomic32_sub(&rkb->rkb_rk->rk_broker_down_cnt, 1); } rd_atomic32_sub(&rkb->rkb_rk->rk_broker_cnt, 1); @@ -4765,6 +5137,15 @@ void rd_kafka_broker_destroy_final(rd_kafka_broker_t *rkb) { rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs)); rd_assert(TAILQ_EMPTY(&rkb->rkb_retrybufs.rkbq_bufs)); rd_assert(TAILQ_EMPTY(&rkb->rkb_toppars)); + rd_assert(!rkb->rkb_share_fetch_session.toppars_to_add); + rd_assert(!rkb->rkb_share_fetch_session.toppars_to_forget); + rd_assert(!rkb->rkb_share_async_ack_details); + rd_assert(!rkb->rkb_pending_commit_sync.sync_ack_details); + rd_assert(!rkb->rkb_share_fetch_session.adding_toppars); + rd_assert(!rkb->rkb_share_fetch_session.forgetting_toppars); + rd_assert( + rd_list_empty(rkb->rkb_share_fetch_session.toppars_in_session)); + rd_list_destroy(rkb->rkb_share_fetch_session.toppars_in_session); if (rkb->rkb_source != RD_KAFKA_INTERNAL && (rkb->rkb_rk->rk_conf.security_protocol == @@ -4803,7 +5184,8 @@ void rd_kafka_broker_destroy_final(rd_kafka_broker_t *rkb) { rd_avg_destroy( &rkb->rkb_telemetry.rd_avg_current.rkb_avg_outbuf_latency); - if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER) { + if (rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER && + !RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk)) { rd_avg_destroy( &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_fetch_latency); rd_avg_destroy( @@ -4815,6 +5197,16 @@ void rd_kafka_broker_destroy_final(rd_kafka_broker_t *rkb) { .rkb_avg_produce_latency); } + if (RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk)) { + rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_latency); + rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_current + .rkb_avg_share_fetch_latency); + rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_size); + rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_current + .rkb_avg_share_fetch_size); + } mtx_lock(&rkb->rkb_logname_lock); rd_free(rkb->rkb_logname); @@ -4883,18 +5275,26 @@ rd_kafka_broker_t *rd_kafka_broker_add(rd_kafka_t *rk, rd_snprintf(rkb->rkb_name, sizeof(rkb->rkb_name), "%s", name); } - rkb->rkb_source = source; - rkb->rkb_rk = rk; - rkb->rkb_ts_state = rd_clock(); - rkb->rkb_nodeid = nodeid; - rkb->rkb_proto = proto; - rkb->rkb_port = port; - rkb->rkb_origname = rd_strdup(name); + rkb->rkb_source = source; + rkb->rkb_rk = rk; + rkb->rkb_ts_state = rd_clock(); + rkb->rkb_nodeid = nodeid; + rkb->rkb_proto = proto; + rkb->rkb_port = port; + rkb->rkb_origname = rd_strdup(name); + rkb->rkb_c.connections_max_idle_ms = -1; + rkb->rkb_share_fetch_session.epoch = 0; mtx_init(&rkb->rkb_lock, mtx_plain); mtx_init(&rkb->rkb_logname_lock, mtx_plain); rkb->rkb_logname = rd_strdup(rkb->rkb_name); TAILQ_INIT(&rkb->rkb_toppars); + rkb->rkb_share_fetch_session.toppars_in_session = + rd_list_new(0, rd_kafka_toppar_destroy_free); + rkb->rkb_share_fetch_session.toppars_to_forget = NULL; + rkb->rkb_share_fetch_session.toppars_to_add = NULL; + rkb->rkb_share_async_ack_details = NULL; + rkb->rkb_share_fetch_enqueued = rd_false; CIRCLEQ_INIT(&rkb->rkb_active_toppars); TAILQ_INIT(&rkb->rkb_monitors); rd_kafka_bufq_init(&rkb->rkb_outbufs); @@ -4928,7 +5328,8 @@ rd_kafka_broker_t *rd_kafka_broker_add(rd_kafka_t *rk, RD_AVG_GAUGE, 0, 100 * 1000, 2, rk->rk_conf.enable_metrics_push); - if (rk->rk_type == RD_KAFKA_CONSUMER) { + if (rk->rk_type == RD_KAFKA_CONSUMER && + !RD_KAFKA_IS_SHARE_CONSUMER(rk)) { rd_avg_init( &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_fetch_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, @@ -4946,12 +5347,32 @@ rd_kafka_broker_t *rd_kafka_broker_add(rd_kafka_t *rk, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); } + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_avg_init(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rkb->rkb_telemetry.rd_avg_current + .rkb_avg_share_fetch_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_size, + RD_AVG_GAUGE, 0, 100 * 1024 * 1024, 2, + rk->rk_conf.enable_metrics_push); + rd_avg_init( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size, + RD_AVG_GAUGE, 0, 100 * 1024 * 1024, 2, + rk->rk_conf.enable_metrics_push); + } + rd_refcnt_init(&rkb->rkb_refcnt, 0); rd_kafka_broker_keep(rkb); /* rk_broker's refcount */ rkb->rkb_reconnect_backoff_ms = rk->rk_conf.reconnect_backoff_ms; rd_atomic32_init(&rkb->rkb_persistconn.coord, 0); rd_atomic32_init(&rkb->termination_in_progress, 0); + rd_atomic32_init(&rkb->rkb_down_reported, 0); rd_atomic64_init(&rkb->rkb_c.ts_send, 0); rd_atomic64_init(&rkb->rkb_c.ts_recv, 0); @@ -5670,25 +6091,6 @@ static int rd_kafka_broker_filter_never_connected(rd_kafka_broker_t *rkb, return rd_atomic32_get(&rkb->rkb_c.connects); } -/** - * @brief Filter out brokers that aren't learned ones. - */ -static int rd_kafka_broker_filter_learned(rd_kafka_broker_t *rkb, - void *opaque) { - return rkb->rkb_source != RD_KAFKA_LEARNED; -} - -/** - * @brief Filter out brokers that aren't learned ones or - * that have at least one connection attempt. - */ -static int -rd_kafka_broker_filter_learned_never_connected(rd_kafka_broker_t *rkb, - void *opaque) { - return rd_atomic32_get(&rkb->rkb_c.connects) || - rkb->rkb_source != RD_KAFKA_LEARNED; -} - static void rd_kafka_connect_any_timer_cb(rd_kafka_timers_t *rkts, void *arg) { const char *reason = (const char *)arg; rd_kafka_t *rk = rkts->rkts_rk; @@ -5744,63 +6146,13 @@ void rd_kafka_connect_any(rd_kafka_t *rk, const char *reason) { return; } - /* In case there no learned brokers never connected to, - * 90% of times select a learned broker in init state. - * - * This avoids problems after re-bootstrapping that cause - * the bootstrap brokers to be always preferred - * given there are learned brokers that already connected and - * caused ALL_BROKERS_DOWN. - * - * If that happens those learned brokers - * that already connected are never selected unless - * they disappear and re-appear again as new brokers with 0 connects, - * so we have to assing a higher probability to it. - * - * Additionally we cannot always prefer the learned - * brokers as their address could have changed and we need to - * connect to the bootstrap brokers to know that. - * KIP-1102 `metadata.recovery.rebootstrap.trigger.ms` would - * be triggered in this case after 5 mins - * but it's a long time to wait. - */ - - /* First pass: only match learned brokers never connected to - * in state INIT, to try to exhaust - * the available brokers so that an - * ERR_ALL_BROKERS_DOWN error can be raised. */ - rkb = rd_kafka_broker_random( - rk, RD_KAFKA_BROKER_STATE_INIT, - rd_kafka_broker_filter_learned_never_connected, NULL); - -#if ENABLE_DEVEL == 1 - if (rkb) - rd_dassert(rkb->rkb_source == RD_KAFKA_LEARNED); -#endif - - if (!rkb && rd_jitter(0, 9) > 0) { /* 0.9 probability */ - /* Second pass: only match learned brokers - * in state INIT. */ - rkb = rd_kafka_broker_random(rk, RD_KAFKA_BROKER_STATE_INIT, - rd_kafka_broker_filter_learned, - NULL); - -#if ENABLE_DEVEL == 1 - if (rkb) - rd_dassert(rkb->rkb_source == RD_KAFKA_LEARNED); -#endif - } - - /* Third pass: only match brokers never connected to, + /* First pass: only match brokers never connected to, * to try to exhaust the available brokers - * so that an ERR_ALL_BROKERS_DOWN error - * can be raised. */ - if (!rkb) - rkb = rd_kafka_broker_random( - rk, RD_KAFKA_BROKER_STATE_INIT, - rd_kafka_broker_filter_never_connected, NULL); - - /* Fourth pass: match any non-connected/non-connecting broker. */ + * so that an ERR_ALL_BROKERS_DOWN error can be raised. */ + rkb = rd_kafka_broker_random(rk, RD_KAFKA_BROKER_STATE_INIT, + rd_kafka_broker_filter_never_connected, + NULL); + /* Second pass: match any non-connected/non-connecting broker. */ if (!rkb) rkb = rd_kafka_broker_random(rk, RD_KAFKA_BROKER_STATE_INIT, NULL, NULL); @@ -5889,8 +6241,8 @@ static void rd_kafka_broker_handle_purge_queues(rd_kafka_broker_t *rkb, * the protocol stream, so we need to disconnect from the broker * to get a clean protocol socket. */ if (partial_cnt) - rd_kafka_broker_fail( - rkb, LOG_DEBUG, RD_KAFKA_RESP_ERR__PURGE_QUEUE, + rd_kafka_broker_planned_fail( + rkb, RD_KAFKA_RESP_ERR__PURGE_QUEUE, "Purged %d partially sent request: " "forcing disconnect", partial_cnt); @@ -6258,6 +6610,15 @@ void rd_kafka_broker_decommission(rd_kafka_t *rk, rd_kafka_cgrp_coord_dead(rk->rk_cgrp, RD_KAFKA_RESP_ERR__DESTROY_BROKER, "Group coordinator decommissioned"); + + if (RD_KAFKA_IS_SHARE_CONSUMER(rk) && + rkb->rkb_source == RD_KAFKA_LEARNED) { + rd_kafka_share_acks_clear_during_broker_decommission(rk, rkb); + rd_kafka_q_enq( + rkb->rkb_ops, + rd_kafka_op_new(RD_KAFKA_OP_SHARE_SESSION_CLEAR)); + } + /* Send op to trigger queue/io wake-up. * Broker thread will destroy this thread reference. * WARNING: This is last time we can read from rkb in this thread! */ @@ -6266,6 +6627,98 @@ void rd_kafka_broker_decommission(rd_kafka_t *rk, rd_kafka_wrlock(rk); } +/** + * @brief Unittest for reconnect.backoff.ms + */ +static int rd_ut_reconnect_backoff(void) { + rd_kafka_broker_t rkb = RD_ZERO_INIT; + rd_kafka_conf_t conf = {.reconnect_backoff_ms = 10, + .reconnect_backoff_max_ms = 90}; + rd_ts_t now = 1000000; + int backoff; + + rkb.rkb_reconnect_backoff_ms = conf.reconnect_backoff_ms; + + /* broker's backoff is the initial reconnect.backoff.ms=10 */ + rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); + backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); + RD_UT_ASSERT_RANGE(backoff, 7, 15, "%d"); + + /* .. 20 */ + rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); + backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); + RD_UT_ASSERT_RANGE(backoff, 15, 30, "%d"); + + /* .. 40 */ + rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); + backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); + RD_UT_ASSERT_RANGE(backoff, 30, 60, "%d"); + + /* .. 80, the jitter is capped at reconnect.backoff.max.ms=90 */ + rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); + backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); + RD_UT_ASSERT_RANGE(backoff, 60, conf.reconnect_backoff_max_ms, "%d"); + + /* .. 90, capped by reconnect.backoff.max.ms */ + rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); + backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); + RD_UT_ASSERT_RANGE(backoff, 67, conf.reconnect_backoff_max_ms, "%d"); + + /* .. 90, should remain at capped value. */ + rd_kafka_broker_update_reconnect_backoff(&rkb, &conf, now); + backoff = rd_kafka_broker_reconnect_backoff(&rkb, now); + RD_UT_ASSERT_RANGE(backoff, 67, conf.reconnect_backoff_max_ms, "%d"); + + RD_UT_PASS(); +} + +/** + * @brief Unittest for reconnect.backoff.ms + */ +static int rd_ut_ApiVersion_at_least(void) { + rd_kafka_broker_t rkb = RD_ZERO_INIT; + mtx_init(&rkb.rkb_lock, mtx_plain); + + struct rd_kafka_ApiVersion av = { + .ApiKey = RD_KAFKAP_Metadata, + .MinVer = 5, + .MaxVer = 10, + }; + + rkb.rkb_ApiVersions_cnt = 1; + rkb.rkb_ApiVersions = &av; + + RD_UT_ASSERT( + rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 0), + "Metadata API version should be at least 0"); + RD_UT_ASSERT( + rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 3), + "Metadata API version should be at least 3"); + RD_UT_ASSERT( + rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 5), + "Metadata API version should be at least 5"); + RD_UT_ASSERT( + rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 7), + "Metadata API version should be at least 7"); + RD_UT_ASSERT( + rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 10), + "Metadata API version should be at least 10"); + RD_UT_ASSERT( + !rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 11), + "Metadata API version shouldn't be at least 11"); + + rkb.rkb_ApiVersions_cnt = 0; + RD_UT_ASSERT( + !rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 0), + "Metadata API version shouldn't be at least 0"); + RD_UT_ASSERT( + !rd_kafka_broker_ApiVersion_at_least(&rkb, RD_KAFKAP_Metadata, 3), + "Metadata API version shouldn't be at least 3"); + + mtx_destroy(&rkb.rkb_lock); + RD_UT_PASS(); +} + /** * @name Unit tests * @{ @@ -6275,6 +6728,7 @@ int unittest_broker(void) { int fails = 0; fails += rd_ut_reconnect_backoff(); + fails += rd_ut_ApiVersion_at_least(); return fails; } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_broker.h b/lib/librdkafka-2.15.0/src/rdkafka_broker.h similarity index 76% rename from lib/librdkafka-2.10.1/src/rdkafka_broker.h rename to lib/librdkafka-2.15.0/src/rdkafka_broker.h index 5e8ea82f290..a08815017ab 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_broker.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_broker.h @@ -106,6 +106,61 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */ /* Toppars handled by this broker */ TAILQ_HEAD(, rd_kafka_toppar_s) rkb_toppars; + + /** + * TODO KIP-932: Consider using a map instead of rd_list_t for + * some of the partition lists to optimize + * add/remove performance. + */ + struct { + rd_list_t *toppars_in_session; /* List of toppars in the current + * fetch session. Any new added + * toppar in rkb_toppars will be + * added here after successful + * share fetch request. Any + * removed toppar from + * rkb_toppars will be removed + * from here after successful + * share fetch request. */ + rd_list_t + *toppars_to_add; /* TODO KIP-932: Consider using a map + * for performance improvements. List + * of toppars that are to be added to + * the fetch session. `adding_toppars` + * are removed from this when fetch + * request is successful. */ + + rd_list_t + *adding_toppars; /* List of toppars that are being added to + * the session. These are already sent in + * the fetch request. Will be removed from + * `toppars_to_add` when fetch request is + * successful. This is cleared and set to + * NULL after the response. + */ + + rd_list_t + *toppars_to_forget; /* TODO KIP-932: Consider using a map + * for performance improvements. List + * of toppars that are removed from + * the session. `forgetting_toppars` + * are removed from this when fetch + * request is successful. */ + + rd_list_t *forgetting_toppars; /* List of toppars that are being + * removed from the session. + * These are already sent in the + * fetch request. Will be removed + * from `toppars_to_forget` when + * fetch request is successful. + * This is cleared and set to + * NULL after the response. + */ + int32_t epoch; /* Current fetch session + * epoch, or -1 if leaving the session + */ + } rkb_share_fetch_session; + int rkb_toppar_cnt; /* Active toppars that are eligible for: @@ -187,8 +242,27 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */ rd_atomic64_t reqtype[RD_KAFKAP__NUM]; /**< Per request-type * counter */ - rd_atomic64_t ts_send; /**< Timestamp of last send */ - rd_atomic64_t ts_recv; /**< Timestamp of last receive */ + rd_atomic64_t ts_send; /**< Timestamp of last send */ + rd_atomic64_t ts_recv; /**< Timestamp of last receive */ + rd_bool_t skip_broker_down; /**< Avoid reporting the + * broker down on next + * state change. + * Useful for a planned + * disconnection to avoid + * reaching the all + * brokers down state. */ + int connections_max_idle_ms; /**< Maximum idle time + * for this broker connections. + * jitter is different for + * each broker to avoid the + * ALL_BROKERS_DOWN error. + * Initial value is `-1` as + * `connections.max.idle.ms=0` + * means disabled, but must + * never be used when + * `connections.max.idle.ms=0` + * for the same reason + * (an assert is present). */ } rkb_c; struct { @@ -201,12 +275,19 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */ rd_avg_t rkb_avg_rtt; /* Current RTT avg */ rd_avg_t rkb_avg_throttle; /* Current throttle avg */ rd_avg_t - rkb_avg_outbuf_latency; /**< Current latency - * between buf_enq0 - * and writing to socket - */ - rd_avg_t rkb_avg_fetch_latency; /**< Current fetch - * latency avg */ + rkb_avg_outbuf_latency; /**< Current latency + * between buf_enq0 + * and writing to socket + */ + rd_avg_t rkb_avg_fetch_latency; /**< Current fetch + * latency avg */ + rd_avg_t rkb_avg_share_fetch_latency; /**< Current share + * fetch latency + * avg */ + rd_avg_t rkb_avg_share_fetch_size; /**< Current share + * fetch response + * size avg + * (bytes) */ rd_avg_t rkb_avg_produce_latency; /**< Current produce * latency avg */ } rd_avg_current; @@ -219,6 +300,14 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */ * latency avg */ rd_avg_t rkb_avg_fetch_latency; /**< Rolled over fetch * latency avg */ + rd_avg_t + rkb_avg_share_fetch_latency; /**< Rolled over + * share fetch + * latency avg */ + rd_avg_t + rkb_avg_share_fetch_size; /**< Rolled over + * share fetch response + * size avg (bytes) */ rd_avg_t rkb_avg_produce_latency; /**< Rolled over produce * latency avg */ @@ -331,7 +420,9 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */ rd_kafka_secproto_t rkb_proto; - int rkb_down_reported; /* Down event reported */ + /** Down event was reported for this broker + * after last connection to any broker. */ + rd_atomic32_t rkb_down_reported; #if WITH_SASL_CYRUS rd_kafka_timer_t rkb_sasl_kinit_refresh_tmr; #endif @@ -366,6 +457,43 @@ struct rd_kafka_broker_s { /* rd_kafka_broker_t */ /** > 0 if this broker thread is terminating */ rd_atomic32_t termination_in_progress; + + /** + * Whether a share fetch should_fetch set is enqueued on + * this broker's op queue or not. + */ + rd_bool_t rkb_share_fetch_enqueued; + + rd_list_t + *rkb_share_async_ack_details; /**< Pending ack batches for + * this broker (as partition + * leader). Type: + * rd_kafka_share_ack_batches_t*. + * Allocated by main thread + * FANOUT handler, moved to + * SHARE_FETCH op and set to + * NULL. Freed by broker + * thread after use. + * @locality main thread */ + + /** + * Pending commit_sync ack details for this broker. + * Stored when a commit_sync request arrives but the broker + * already has an inflight request. Takes priority over + * rkb_share_async_ack_details when dispatching. + * @locality main thread + */ + struct { + rd_list_t *sync_ack_details; /**< Ack batches waiting to be + * sent. Type: + * rd_kafka_share_ack_batches_t*. + */ + rd_ts_t abs_timeout; /**< Absolute timeout from + * the commit_sync request. */ + int64_t commit_sync_request_id; /**< Request ID this + * pending data + * belongs to. */ + } rkb_pending_commit_sync; }; #define rd_kafka_broker_keep(rkb) rd_refcnt_add(&(rkb)->rkb_refcnt) @@ -477,6 +605,14 @@ int16_t rd_kafka_broker_ApiVersion_supported0(rd_kafka_broker_t *rkb, int *featuresp, rd_bool_t do_lock); +rd_bool_t rd_kafka_broker_ApiVersion_at_least(rd_kafka_broker_t *rkb, + int16_t ApiKey, + int16_t minver); + +rd_bool_t rd_kafka_broker_ApiVersion_at_least_no_lock(rd_kafka_broker_t *rkb, + int16_t ApiKey, + int16_t minver); + rd_kafka_broker_t *rd_kafka_broker_find_by_nodeid0_fl(const char *func, int line, rd_kafka_t *rk, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_buf.c b/lib/librdkafka-2.15.0/src/rdkafka_buf.c similarity index 98% rename from lib/librdkafka-2.10.1/src/rdkafka_buf.c rename to lib/librdkafka-2.15.0/src/rdkafka_buf.c index fca0cc28b61..012835de088 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_buf.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_buf.c @@ -241,6 +241,12 @@ void rd_kafka_bufq_init(rd_kafka_bufq_t *rkbufq) { rd_atomic32_init(&rkbufq->rkbq_msg_cnt, 0); } +static void rd_kafka_bufq_reset(rd_kafka_bufq_t *rkbufq) { + TAILQ_INIT(&rkbufq->rkbq_bufs); + rd_atomic32_set(&rkbufq->rkbq_cnt, 0); + rd_atomic32_set(&rkbufq->rkbq_msg_cnt, 0); +} + /** * Concat all buffers from 'src' to tail of 'dst' */ @@ -249,7 +255,7 @@ void rd_kafka_bufq_concat(rd_kafka_bufq_t *dst, rd_kafka_bufq_t *src) { (void)rd_atomic32_add(&dst->rkbq_cnt, rd_atomic32_get(&src->rkbq_cnt)); (void)rd_atomic32_add(&dst->rkbq_msg_cnt, rd_atomic32_get(&src->rkbq_msg_cnt)); - rd_kafka_bufq_init(src); + rd_kafka_bufq_reset(src); } /** diff --git a/lib/librdkafka-2.10.1/src/rdkafka_buf.h b/lib/librdkafka-2.15.0/src/rdkafka_buf.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_buf.h rename to lib/librdkafka-2.15.0/src/rdkafka_buf.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_cert.c b/lib/librdkafka-2.15.0/src/rdkafka_cert.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_cert.c rename to lib/librdkafka-2.15.0/src/rdkafka_cert.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_cert.h b/lib/librdkafka-2.15.0/src/rdkafka_cert.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_cert.h rename to lib/librdkafka-2.15.0/src/rdkafka_cert.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_cgrp.c b/lib/librdkafka-2.15.0/src/rdkafka_cgrp.c similarity index 88% rename from lib/librdkafka-2.10.1/src/rdkafka_cgrp.c rename to lib/librdkafka-2.15.0/src/rdkafka_cgrp.c index 623005c559c..be024e478e0 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_cgrp.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_cgrp.c @@ -37,6 +37,7 @@ #include "rdkafka_metadata.h" #include "rdkafka_cgrp.h" #include "rdkafka_interceptor.h" +#include "rdkafka_share_acknowledgement.h" #include "rdmap.h" #include "rdunittest.h" @@ -555,12 +556,6 @@ rd_kafka_cgrp_t *rd_kafka_cgrp_new(rd_kafka_t *rk, rk->rk_conf.auto_commit_interval_ms * 1000ll, rd_kafka_cgrp_offset_commit_tmr_cb, rkcg); - if (rkcg->rkcg_group_protocol == RD_KAFKA_GROUP_PROTOCOL_CONSUMER) { - rd_kafka_log(rk, LOG_WARNING, "CGRP", - "KIP-848 Consumer Group Protocol is in 'Preview' " - "and MUST NOT be used in production"); - } - return rkcg; } @@ -654,6 +649,9 @@ static int rd_kafka_cgrp_coord_update(rd_kafka_cgrp_t *rkcg, int32_t coord_id) { /* Clear previous broker handle, if any */ if (rkcg->rkcg_curr_coord) rd_kafka_cgrp_coord_clear_broker(rkcg); + + rd_kafka_cgrp_consumer_expedite_next_heartbeat( + rkcg, "coordinator changed"); } @@ -788,6 +786,8 @@ static void rd_kafka_cgrp_handle_FindCoordinator(rd_kafka_t *rk, RD_KAFKA_ERR_ACTION_RETRY, RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE, + RD_KAFKA_ERR_ACTION_RETRY, RD_KAFKA_RESP_ERR__DESTROY_BROKER, + RD_KAFKA_ERR_ACTION_END); @@ -807,6 +807,12 @@ static void rd_kafka_cgrp_handle_FindCoordinator(rd_kafka_t *rk, rd_kafka_cgrp_set_last_err(rkcg, ErrorCode); } + if (ErrorCode == RD_KAFKA_RESP_ERR__DESTROY_BROKER) { + /* This error is one-time and should cause + * an immediate retry. */ + rd_interval_reset(&rkcg->rkcg_coord_query_intvl); + } + /* Retries are performed by the timer-intervalled * coord queries, continue querying */ rd_kafka_cgrp_set_state(rkcg, RD_KAFKA_CGRP_STATE_QUERY_COORD); @@ -976,12 +982,12 @@ static void rd_kafka_cgrp_handle_LeaveGroup(rd_kafka_t *rk, err: if (ErrorCode) - rd_kafka_dbg(rkb->rkb_rk, CGRP, "LEAVEGROUP", + rd_kafka_dbg(rk, CGRP, "LEAVEGROUP", "LeaveGroup response error in state %s: %s", rd_kafka_cgrp_state_names[rkcg->rkcg_state], rd_kafka_err2str(ErrorCode)); else - rd_kafka_dbg(rkb->rkb_rk, CGRP, "LEAVEGROUP", + rd_kafka_dbg(rk, CGRP, "LEAVEGROUP", "LeaveGroup response received in state %s", rd_kafka_cgrp_state_names[rkcg->rkcg_state]); @@ -1075,7 +1081,94 @@ rd_kafka_cgrp_handle_ConsumerGroupHeartbeat_leave(rd_kafka_t *rk, goto err; } +static void +rd_kafka_cgrp_handle_ShareGroupHeartbeat_leave(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request, + void *opaque) { + rd_kafka_cgrp_t *rkcg = opaque; + const int log_decode_errors = LOG_ERR; + int16_t ErrorCode = 0; + + if (err) { + ErrorCode = err; + goto err; + } + + /* Count the leave ShareGroupHeartbeat response for + * consumer.share.coordinator.heartbeat.{total,rate} */ + rd_atomic64_add(&rk->rk_telemetry.heartbeat_total, 1); + + rd_kafka_buf_read_throttle_time(rkbuf); + + rd_kafka_buf_read_i16(rkbuf, &ErrorCode); +err: + if (ErrorCode) + rd_kafka_dbg( + rkb->rkb_rk, CGRP, "LEAVEGROUP", + "ShareGroupHeartbeat response error in state %s: %s", + rd_kafka_cgrp_state_names[rkcg->rkcg_state], + rd_kafka_err2str(ErrorCode)); + else + rd_kafka_dbg( + rkb->rkb_rk, CGRP, "LEAVEGROUP", + "ShareGroupHeartbeat response received in state %s", + rd_kafka_cgrp_state_names[rkcg->rkcg_state]); + rd_kafka_cgrp_consumer_reset(rkcg); + if (ErrorCode != RD_KAFKA_RESP_ERR__DESTROY) { + rd_assert(thrd_is_current(rk->rk_thread)); + rkcg->rkcg_flags &= ~RD_KAFKA_CGRP_F_WAIT_LEAVE; + rd_kafka_cgrp_try_terminate(rkcg); + } + return; +err_parse: + ErrorCode = rkbuf->rkbuf_err; + goto err; +} + +static void rd_kafka_cgrp_share_consumer_leave(rd_kafka_cgrp_t *rkcg) { + int32_t member_epoch = -1; + + if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_LEAVE) { + rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "LEAVE", + "Group \"%.*s\": leave (in state %s): " + "ShareGroupHeartbeat already in-transit", + RD_KAFKAP_STR_PR(rkcg->rkcg_group_id), + rd_kafka_cgrp_state_names[rkcg->rkcg_state]); + return; + } + + rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "LEAVE", + "Group \"%.*s\": leave (in state %s)", + RD_KAFKAP_STR_PR(rkcg->rkcg_group_id), + rd_kafka_cgrp_state_names[rkcg->rkcg_state]); + + rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_WAIT_LEAVE; + + if (rkcg->rkcg_state == RD_KAFKA_CGRP_STATE_UP) { + rd_rkb_dbg(rkcg->rkcg_curr_coord, CONSUMER, "LEAVE", + "Share consumer: leaving group"); + rd_kafka_ShareGroupHeartbeatRequest( + rkcg->rkcg_coord, rkcg->rkcg_group_id, rkcg->rkcg_member_id, + member_epoch, NULL /* no rack */, + NULL /* no subscription topics */, + RD_KAFKA_REPLYQ(rkcg->rkcg_ops, 0), + rd_kafka_cgrp_handle_ShareGroupHeartbeat_leave, rkcg); + } else { + rd_kafka_cgrp_handle_ShareGroupHeartbeat_leave( + rkcg->rkcg_rk, rkcg->rkcg_coord, + RD_KAFKA_RESP_ERR__WAIT_COORD, NULL, NULL, rkcg); + } +} + static void rd_kafka_cgrp_consumer_leave(rd_kafka_cgrp_t *rkcg) { + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) { + rd_kafka_cgrp_share_consumer_leave(rkcg); + return; + } + int32_t member_epoch = -1; if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_LEAVE) { @@ -3074,8 +3167,6 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, rd_dassert(rkcg->rkcg_flags & RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT); - if (rd_kafka_cgrp_will_leave(rkcg)) - err = RD_KAFKA_RESP_ERR__OUTDATED; if (err) goto err; @@ -3095,15 +3186,25 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, } rd_kafka_buf_read_i32(rkbuf, &member_epoch); - rd_kafka_buf_read_i32(rkbuf, &heartbeat_interval_ms); - - int8_t are_assignments_present; - rd_kafka_buf_read_i8(rkbuf, &are_assignments_present); rkcg->rkcg_generation_id = member_epoch; + + rd_kafka_dbg(rk, CGRP, "HEARTBEAT", + "ConsumerGroupHeartbeat response received for " + "member id \"%.*s\" with epoch %d", + RD_KAFKAP_STR_PR(&member_id), member_epoch); + + if (rd_kafka_cgrp_will_leave(rkcg)) { + err = RD_KAFKA_RESP_ERR__OUTDATED; + goto err; + } + + rd_kafka_buf_read_i32(rkbuf, &heartbeat_interval_ms); if (heartbeat_interval_ms > 0) { rkcg->rkcg_heartbeat_intvl_ms = heartbeat_interval_ms; } + int8_t are_assignments_present; + rd_kafka_buf_read_i8(rkbuf, &are_assignments_present); if (are_assignments_present == 1) { rd_kafka_topic_partition_list_t *assigned_topic_partitions; const rd_kafka_topic_partition_field_t assignments_fields[] = { @@ -3123,11 +3224,10 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, sizeof(assigned_topic_partitions_str), 0); } - rd_kafka_dbg( - rk, CGRP, "HEARTBEAT", - "ConsumerGroupHeartbeat response received target " - "assignment \"%s\"", - assigned_topic_partitions_str); + rd_kafka_dbg(rk, CGRP, "HEARTBEAT", + "ConsumerGroupHeartbeat received target " + "assignment \"%s\"", + assigned_topic_partitions_str); } if (assigned_topic_partitions) { @@ -3276,6 +3376,14 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, RD_KAFKA_CGRP_CONSUMER_F_WAIT_REJOIN; return; + case RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND: + /* If the leave heartbeat has already been sent the member is + * on its way out of the group, so a GROUP_ID_NOT_FOUND on a + * still-in-flight regular heartbeat is expected and benign. + * Otherwise the group is genuinely gone and this is fatal. */ + if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_LEAVE) + return; + /* FALLTHRU */ case RD_KAFKA_RESP_ERR_INVALID_REQUEST: case RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED: case RD_KAFKA_RESP_ERR_UNSUPPORTED_ASSIGNOR: @@ -3285,6 +3393,10 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, case RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED: actions = RD_KAFKA_ERR_ACTION_FATAL; break; + /* FIXME: GROUP_ID_NOT_FOUND on a non-leave heartbeat indicates the + * coordinator no longer knows the group and should be treated as a + * fatal error here; it currently falls through to the default action. + */ default: actions = rd_kafka_err_action( rkb, err, request, @@ -3293,6 +3405,12 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED, RD_KAFKA_ERR_ACTION_END); + + if (actions & RD_KAFKA_ERR_ACTION_PERMANENT && + err >= RD_KAFKA_RESP_ERR_UNKNOWN) { + actions |= RD_KAFKA_ERR_ACTION_FATAL; + } + break; } @@ -3320,8 +3438,7 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, rkcg->rkcg_consumer_flags |= RD_KAFKA_CGRP_CONSUMER_F_SEND_FULL_REQUEST; rd_kafka_cgrp_coord_query(rkcg, rd_kafka_err2str(err)); - rd_kafka_cgrp_consumer_expedite_next_heartbeat( - rkcg, "coordinator query"); + /* If coordinator changes, HB will be expedited. */ } if (actions & RD_KAFKA_ERR_ACTION_SPECIAL) { @@ -3333,6 +3450,11 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, (rd_clock() > rkcg->rkcg_ts_last_err + min_error_interval)) { rd_kafka_cgrp_set_last_err(rkcg, err); + rd_kafka_log(rkcg->rkcg_rk, LOG_ERR, "HEARTBEAT", + "ConsumerGroupHeartbeat failed: %s%s%.*s", + rd_kafka_err2str(err), + RD_KAFKAP_STR_LEN(&error_str) ? ": " : "", + RD_KAFKAP_STR_PR(&error_str)); rd_kafka_consumer_err( rkcg->rkcg_q, rd_kafka_broker_id(rkb), err, 0, NULL, NULL, err, @@ -3352,6 +3474,330 @@ void rd_kafka_cgrp_handle_ConsumerGroupHeartbeat(rd_kafka_t *rk, } } +void rd_kafka_cgrp_handle_ShareGroupHeartbeat(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request, + void *opaque) { + rd_kafka_cgrp_t *rkcg = rk->rk_cgrp; + const int log_decode_errors = LOG_ERR; + int16_t error_code = 0; + int actions = 0; + rd_kafkap_str_t error_str = RD_KAFKAP_STR_INITIALIZER_EMPTY; + rd_kafkap_str_t member_id; + int32_t member_epoch; + int32_t heartbeat_interval_ms; + int8_t are_assignments_present; + + if (err == RD_KAFKA_RESP_ERR__DESTROY) + return; + + rd_dassert(rkcg->rkcg_flags & RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT); + + if (err) + goto err; + + /* Count this ShareGroupHeartbeat response for + * consumer.share.coordinator.heartbeat.{total,rate}. + * recorded once the response arrives (transport-level success), + * regardless of Broker level errors */ + rd_atomic64_add(&rk->rk_telemetry.heartbeat_total, 1); + + rd_kafka_buf_read_throttle_time(rkbuf); + + rd_kafka_buf_read_i16(rkbuf, &error_code); + rd_kafka_buf_read_str(rkbuf, &error_str); + + if (error_code) { + err = error_code; + goto err; + } + + rd_kafka_buf_read_str(rkbuf, &member_id); + if (!RD_KAFKAP_STR_IS_NULL(&member_id)) { + rd_kafka_cgrp_set_member_id(rkcg, member_id.str); + } + + rd_kafka_buf_read_i32(rkbuf, &member_epoch); + rkcg->rkcg_generation_id = member_epoch; + + if (rd_kafka_cgrp_will_leave(rkcg)) { + err = RD_KAFKA_RESP_ERR__OUTDATED; + goto err; + } + + rd_kafka_buf_read_i32(rkbuf, &heartbeat_interval_ms); + if (heartbeat_interval_ms > 0) { + rkcg->rkcg_heartbeat_intvl_ms = heartbeat_interval_ms; + } + + rd_kafka_buf_read_i8(rkbuf, &are_assignments_present); + + if (are_assignments_present == 1) { + rd_kafka_topic_partition_list_t *assigned_topic_partitions; + const rd_kafka_topic_partition_field_t assignments_fields[] = { + RD_KAFKA_TOPIC_PARTITION_FIELD_PARTITION, + RD_KAFKA_TOPIC_PARTITION_FIELD_END}; + assigned_topic_partitions = rd_kafka_buf_read_topic_partitions( + rkbuf, rd_true, rd_false /* Don't use Topic Name */, 0, + assignments_fields); + + rd_kafka_dbg(rk, CGRP, "HEARTBEAT", + "ShareGroupHeartbeat response received " + "assigned_topic_partitions size %d", + assigned_topic_partitions->cnt); + + if (rd_kafka_is_dbg(rk, CGRP)) { + char assigned_topic_partitions_str[512] = "NULL"; + + if (assigned_topic_partitions) { + rd_kafka_topic_partition_list_str( + assigned_topic_partitions, + assigned_topic_partitions_str, + sizeof(assigned_topic_partitions_str), 0); + } + + rd_kafka_dbg( + rk, CGRP, "HEARTBEAT", + "ShareGroupHeartbeat response received target " + "assignment \"%s\"", + assigned_topic_partitions_str); + } + + if (assigned_topic_partitions) { + RD_IF_FREE(rkcg->rkcg_next_target_assignment, + rd_kafka_topic_partition_list_destroy); + rkcg->rkcg_next_target_assignment = NULL; + if (rd_kafka_cgrp_consumer_is_new_assignment_different( + rkcg, assigned_topic_partitions)) { + rkcg->rkcg_next_target_assignment = + assigned_topic_partitions; + } else { + rd_kafka_topic_partition_list_destroy( + assigned_topic_partitions); + assigned_topic_partitions = NULL; + } + } + } + + if (rkcg->rkcg_join_state == RD_KAFKA_CGRP_JOIN_STATE_STEADY && + (rkcg->rkcg_consumer_flags & RD_KAFKA_CGRP_CONSUMER_F_WAIT_ACK) && + rkcg->rkcg_target_assignment) { + if (rkcg->rkcg_consumer_flags & + RD_KAFKA_CGRP_CONSUMER_F_SENDING_ACK) { + if (rkcg->rkcg_current_assignment) + rd_kafka_topic_partition_list_destroy( + rkcg->rkcg_current_assignment); + rkcg->rkcg_current_assignment = + rd_kafka_topic_partition_list_copy( + rkcg->rkcg_target_assignment); + rd_kafka_topic_partition_list_destroy( + rkcg->rkcg_target_assignment); + rkcg->rkcg_target_assignment = NULL; + rkcg->rkcg_consumer_flags &= + ~RD_KAFKA_CGRP_CONSUMER_F_WAIT_ACK; + + if (rd_kafka_is_dbg(rkcg->rkcg_rk, CGRP)) { + char rkcg_current_assignment_str[512] = "NULL"; + + rd_kafka_topic_partition_list_str( + rkcg->rkcg_current_assignment, + rkcg_current_assignment_str, + sizeof(rkcg_current_assignment_str), 0); + + rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "HEARTBEAT", + "Target assignment acked, new " + "current assignment " + " \"%s\"", + rkcg_current_assignment_str); + } + } else if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_SUBSCRIPTION) { + /* We've finished reconciliation but we weren't + * sending an ack, need to send a new HB with the ack. + */ + rd_kafka_cgrp_consumer_expedite_next_heartbeat( + rkcg, "not subscribed anymore"); + } + } + + + if (rkcg->rkcg_consumer_flags & + RD_KAFKA_CGRP_CONSUMER_F_SERVE_PENDING && + rkcg->rkcg_join_state == RD_KAFKA_CGRP_JOIN_STATE_STEADY) { + /* TODO: Check if this should be done only for the + * steady state? + */ + rd_kafka_assignment_serve(rk); + rkcg->rkcg_consumer_flags &= + ~RD_KAFKA_CGRP_CONSUMER_F_SERVE_PENDING; + } + + if (rkcg->rkcg_next_target_assignment) { + if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_SUBSCRIPTION) { + rd_kafka_cgrp_consumer_next_target_assignment_request_metadata( + rk, rkb); + } else { + /* Consumer left the group sending an HB request + * while this one was in-flight. */ + rd_kafka_topic_partition_list_destroy( + rkcg->rkcg_next_target_assignment); + rkcg->rkcg_next_target_assignment = NULL; + } + } + + if (rd_kafka_cgrp_consumer_subscription_preconditions_met(rkcg)) + rd_kafka_cgrp_consumer_expedite_next_heartbeat( + rkcg, "send new subscription"); + + rkcg->rkcg_consumer_flags &= + ~RD_KAFKA_CGRP_CONSUMER_F_SENDING_NEW_SUBSCRIPTION & + ~RD_KAFKA_CGRP_CONSUMER_F_SEND_FULL_REQUEST; + rd_kafka_cgrp_maybe_clear_heartbeat_failed_err(rkcg); + rkcg->rkcg_last_heartbeat_err = RD_KAFKA_RESP_ERR_NO_ERROR; + rkcg->rkcg_expedite_heartbeat_retries = 0; + rkcg->rkcg_flags &= ~RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT; + + return; + +err_parse: + err = rkbuf->rkbuf_err; +err: + rkcg->rkcg_last_heartbeat_err = err; + rkcg->rkcg_flags &= ~RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT; + switch (err) { + case RD_KAFKA_RESP_ERR__DESTROY: + /* quick cleanup */ + return; + + case RD_KAFKA_RESP_ERR_COORDINATOR_LOAD_IN_PROGRESS: + rd_kafka_dbg( + rkcg->rkcg_rk, CONSUMER, "HEARTBEAT", + "ShareGroupHeartbeat failed due to coordinator (%s) " + "loading in progress: %s: " + "retrying", + rkcg->rkcg_curr_coord + ? rd_kafka_broker_name(rkcg->rkcg_curr_coord) + : "none", + rd_kafka_err2str(err)); + actions = RD_KAFKA_ERR_ACTION_RETRY; + break; + + case RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP: + case RD_KAFKA_RESP_ERR_GROUP_COORDINATOR_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR__TRANSPORT: + rd_kafka_dbg( + rkcg->rkcg_rk, CONSUMER, "HEARTBEAT", + "ShareGroupHeartbeat failed due to coordinator (%s) " + "no longer available: %s: " + "re-querying for coordinator", + rkcg->rkcg_curr_coord + ? rd_kafka_broker_name(rkcg->rkcg_curr_coord) + : "none", + rd_kafka_err2str(err)); + /* Remain in joined state and keep querying for coordinator */ + actions = RD_KAFKA_ERR_ACTION_REFRESH; + break; + + case RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID: + case RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH: + rd_kafka_dbg(rkcg->rkcg_rk, CONSUMER, "HEARTBEAT", + "ShareGroupHeartbeat failed due to: %s: " + "will rejoin the group", + rd_kafka_err2str(err)); + rkcg->rkcg_consumer_flags |= + RD_KAFKA_CGRP_CONSUMER_F_WAIT_REJOIN; + return; + + case RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND: + /* If the leave heartbeat has already been sent the member is + * on its way out of the group, so a GROUP_ID_NOT_FOUND on a + * still-in-flight regular heartbeat is expected and benign. + * Otherwise the group is genuinely gone and this is fatal. */ + if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_LEAVE) + return; + /* FALLTHRU */ + case RD_KAFKA_RESP_ERR_INVALID_REQUEST: + case RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED: + case RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION: + case RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE: + case RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED: + actions = RD_KAFKA_ERR_ACTION_FATAL; + break; + + default: + actions = rd_kafka_err_action( + rkb, err, request, + + RD_KAFKA_ERR_ACTION_SPECIAL, + RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED, + + RD_KAFKA_ERR_ACTION_END); + + if (actions & RD_KAFKA_ERR_ACTION_PERMANENT && + err >= RD_KAFKA_RESP_ERR_UNKNOWN) { + actions |= RD_KAFKA_ERR_ACTION_FATAL; + } + break; + } + + if (actions & RD_KAFKA_ERR_ACTION_FATAL) { + rd_kafka_set_fatal_error(rkcg->rkcg_rk, err, + "ShareGroupHeartbeat fatal error: %s", + rd_kafka_err2str(err)); + rd_kafka_cgrp_revoke_all_rejoin_maybe( + rkcg, rd_true, /*assignments lost*/ + rd_true, /*initiating*/ + "Fatal error in ShareGroupHeartbeat API response"); + return; + } + + if (!rkcg->rkcg_heartbeat_intvl_ms) { + /* When an error happens on first HB, it should be always + * retried, unless fatal, to avoid entering a tight loop + * and to use exponential backoff. */ + actions |= RD_KAFKA_ERR_ACTION_RETRY; + } + + if (actions & RD_KAFKA_ERR_ACTION_REFRESH) { + /* Re-query for coordinator */ + rkcg->rkcg_consumer_flags |= + RD_KAFKA_CGRP_CONSUMER_F_SEND_FULL_REQUEST; + rd_kafka_cgrp_coord_query(rkcg, rd_kafka_err2str(err)); + /* If coordinator changes, HB will be expedited. */ + } + + if (actions & RD_KAFKA_ERR_ACTION_SPECIAL) { + rd_ts_t min_error_interval = + RD_MAX(rkcg->rkcg_heartbeat_intvl_ms * 1000, + /* default group.consumer.heartbeat.interval.ms */ + 5000000); + if (rkcg->rkcg_last_err != err || + (rd_clock() > + rkcg->rkcg_ts_last_err + min_error_interval)) { + rd_kafka_cgrp_set_last_err(rkcg, err); + rd_kafka_log(rkcg->rkcg_rk, LOG_ERR, "HEARTBEAT", + "ShareGroupHeartbeat failed: %s%s%.*s", + rd_kafka_err2str(err), + RD_KAFKAP_STR_LEN(&error_str) ? ": " : "", + RD_KAFKAP_STR_PR(&error_str)); + rd_kafka_consumer_err( + rkcg->rkcg_q, rd_kafka_broker_id(rkb), err, 0, NULL, + NULL, err, "ShareGroupHeartbeat failed: %s%s%.*s", + rd_kafka_err2str(err), + RD_KAFKAP_STR_LEN(&error_str) ? ": " : "", + RD_KAFKAP_STR_PR(&error_str)); + } + } + + if (actions & RD_KAFKA_ERR_ACTION_RETRY && + rkcg->rkcg_flags & RD_KAFKA_CGRP_F_SUBSCRIPTION && + !rd_kafka_cgrp_will_leave(rkcg) && + rd_kafka_buf_retry(rkb, request)) { + /* Retry */ + rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT; + } +} /** * @brief Handle Heartbeat response. @@ -3526,6 +3972,11 @@ static void rd_kafka_cgrp_terminated(rd_kafka_cgrp_t *rkcg) { rd_kafka_assert(NULL, !rkcg->rkcg_group_assignment); rd_kafka_assert(NULL, rkcg->rkcg_rk->rk_consumer.wait_commit_cnt == 0); rd_kafka_assert(NULL, rkcg->rkcg_state == RD_KAFKA_CGRP_STATE_TERM); + rd_kafka_assert( + NULL, rkcg->rkcg_share.share_session_leave_remaining_cnt == 0); + rd_kafka_assert( + NULL, rkcg->rkcg_share.share_should_fetch_ops_in_flight_cnt == 0 || + rd_kafka_destroy_flags_no_consumer_close(rkcg->rkcg_rk)); rd_kafka_timer_stop(&rkcg->rkcg_rk->rk_timers, &rkcg->rkcg_offset_commit_tmr, 1 /*lock*/); @@ -3563,6 +4014,9 @@ static void rd_kafka_cgrp_terminated(rd_kafka_cgrp_t *rkcg) { /* Remove cgrp application queue forwarding, if any. */ rd_kafka_q_fwd_set(rkcg->rkcg_q, NULL); + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) { + rd_kafka_q_fwd_set(rkcg->rkcg_rk->rk_rep, NULL); + } /* Destroy KIP-848 consumer group structures */ rd_kafka_cgrp_consumer_reset(rkcg); @@ -3608,6 +4062,7 @@ static RD_INLINE int rd_kafka_cgrp_try_terminate(rd_kafka_cgrp_t *rkcg) { rd_list_empty(&rkcg->rkcg_toppars) && !rd_kafka_assignment_in_progress(rkcg->rkcg_rk) && rkcg->rkcg_rk->rk_consumer.wait_commit_cnt == 0 && + rkcg->rkcg_share.share_session_leave_remaining_cnt == 0 && !(rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_LEAVE)) { /* Since we might be deep down in a 'rko' handler * called from cgrp_op_serve() we cant call terminated() @@ -3624,7 +4079,9 @@ static RD_INLINE int rd_kafka_cgrp_try_terminate(rd_kafka_cgrp_t *rkcg) { "Group \"%s\": " "waiting for %s%d toppar(s), " "%s" - "%d commit(s)%s%s%s (state %s, join-state %s) " + "%d commit(s), " + "%d share session leave(s) remaining" + "%s%s%s (state %s, join-state %s) " "before terminating", rkcg->rkcg_group_id->str, RD_KAFKA_CGRP_WAIT_ASSIGN_CALL(rkcg) ? "assign call, " : "", @@ -3633,6 +4090,7 @@ static RD_INLINE int rd_kafka_cgrp_try_terminate(rd_kafka_cgrp_t *rkcg) { ? "assignment in progress, " : "", rkcg->rkcg_rk->rk_consumer.wait_commit_cnt, + rkcg->rkcg_share.share_session_leave_remaining_cnt, (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_LEAVE) ? ", wait-leave," : "", @@ -3652,8 +4110,8 @@ static RD_INLINE int rd_kafka_cgrp_try_terminate(rd_kafka_cgrp_t *rkcg) { * * @locks none */ -static void rd_kafka_cgrp_partition_add(rd_kafka_cgrp_t *rkcg, - rd_kafka_toppar_t *rktp) { +void rd_kafka_cgrp_partition_add(rd_kafka_cgrp_t *rkcg, + rd_kafka_toppar_t *rktp) { rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "PARTADD", "Group \"%s\": add %s [%" PRId32 "]", rkcg->rkcg_group_id->str, rktp->rktp_rkt->rkt_topic->str, @@ -3662,6 +4120,22 @@ static void rd_kafka_cgrp_partition_add(rd_kafka_cgrp_t *rkcg, rd_kafka_toppar_lock(rktp); rd_assert(!(rktp->rktp_flags & RD_KAFKA_TOPPAR_F_ON_CGRP)); rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_ON_CGRP; + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk) && + rktp->rktp_flags & RD_KAFKA_TOPPAR_F_ON_RKB) { + rd_kafka_op_t *rko; + rko = rd_kafka_op_new(RD_KAFKA_OP_SHARE_SESSION_PARTITION_ADD); + rko->rko_rktp = + rd_kafka_toppar_keep(rktp); /* refcnt from _add op */ + rd_kafka_dbg( + rkcg->rkcg_rk, CGRP, "SHARESESSPARTCGRPADD", + "Group \"%s\": enqueue partition add for %s [%" PRId32 + "] " + "on broker %s", + rkcg->rkcg_group_id->str, rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition, + rd_kafka_broker_name(rktp->rktp_broker)); + rd_kafka_q_enq(rktp->rktp_broker->rkb_ops, rko); + } rd_kafka_toppar_unlock(rktp); rd_kafka_toppar_keep(rktp); @@ -3673,8 +4147,8 @@ static void rd_kafka_cgrp_partition_add(rd_kafka_cgrp_t *rkcg, * * @locks none */ -static void rd_kafka_cgrp_partition_del(rd_kafka_cgrp_t *rkcg, - rd_kafka_toppar_t *rktp) { +void rd_kafka_cgrp_partition_del(rd_kafka_cgrp_t *rkcg, + rd_kafka_toppar_t *rktp) { rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "PARTDEL", "Group \"%s\": delete %s [%" PRId32 "]", rkcg->rkcg_group_id->str, rktp->rktp_rkt->rkt_topic->str, @@ -3684,6 +4158,24 @@ static void rd_kafka_cgrp_partition_del(rd_kafka_cgrp_t *rkcg, rd_assert(rktp->rktp_flags & RD_KAFKA_TOPPAR_F_ON_CGRP); rktp->rktp_flags &= ~RD_KAFKA_TOPPAR_F_ON_CGRP; + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk) && + rktp->rktp_flags & RD_KAFKA_TOPPAR_F_ON_RKB) { + rd_kafka_op_t *rko; + rko = + rd_kafka_op_new(RD_KAFKA_OP_SHARE_SESSION_PARTITION_REMOVE); + rko->rko_rktp = + rd_kafka_toppar_keep(rktp); /* refcnt from _add op */ + rd_kafka_dbg( + rkcg->rkcg_rk, CGRP, "SHARESESSPARTCGRPDEL", + "Group \"%s\": enqueue partition remove for %s [%" PRId32 + "] " + "on broker %s", + rkcg->rkcg_group_id->str, rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition, + rd_kafka_broker_name(rktp->rktp_broker)); + rd_kafka_q_enq(rktp->rktp_broker->rkb_ops, rko); + } + rd_kafka_toppar_purge_internal_fetch_queue_maybe(rktp); rd_kafka_toppar_unlock(rktp); @@ -4646,6 +5138,40 @@ static map_toppar_member_info_t *rd_kafka_toppar_list_to_toppar_member_info_map( } +/** + * @brief Same as rd_kafka_toppar_list_to_toppar_member_info_map but keyed by + * (topic_id, partition). Used by the share-consumer cooperative + * reconciler where every entry carries a topic_id and a same-name + * topic recreate can put OLD-id and NEW-id entries for the same + * (name, partition) into the diff inputs; the by-id key keeps them + * as distinct partitions so the diff correctly produces a revoke + * for the OLD-id and an add for the NEW-id. + * + * @remark \p rktparlist may be NULL. + */ +static map_toppar_member_info_t * +rd_kafka_toppar_list_to_toppar_member_info_map_by_id( + rd_kafka_topic_partition_list_t *rktparlist) { + map_toppar_member_info_t *map = rd_calloc(1, sizeof(*map)); + const rd_kafka_topic_partition_t *rktpar; + + RD_MAP_INIT(map, rktparlist ? rktparlist->cnt : 0, + rd_kafka_topic_partition_by_id_cmp, + rd_kafka_topic_partition_hash_by_id, + rd_kafka_topic_partition_destroy_free, + PartitionMemberInfo_free); + + if (!rktparlist) + return map; + + RD_KAFKA_TPLIST_FOREACH(rktpar, rktparlist) + RD_MAP_SET(map, rd_kafka_topic_partition_copy(rktpar), + PartitionMemberInfo_new(NULL, rd_false)); + + return map; +} + + /** * @brief Construct a toppar list from map \p map with elements corresponding * to the keys of \p map. @@ -4678,11 +5204,26 @@ static void rd_kafka_cgrp_handle_assignment_cooperative( rd_kafka_topic_partition_list_t *newly_added; rd_kafka_topic_partition_list_t *revoked; - new_assignment_set = - rd_kafka_toppar_list_to_toppar_member_info_map(assignment); - - old_assignment_set = rd_kafka_toppar_list_to_toppar_member_info_map( - rkcg->rkcg_group_assignment); + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) { + /* Share-consumer assignment lists always carry topic_id + * (set in rd_kafka_cgrp_consumer_assignment_with_metadata). + * Key the diff maps by (topic_id, partition) so a + * same-name topic recreate produces a revoke for the + * OLD-id and an add for the NEW-id instead of collapsing + * them under a shared (name, partition) bucket. */ + new_assignment_set = + rd_kafka_toppar_list_to_toppar_member_info_map_by_id( + assignment); + old_assignment_set = + rd_kafka_toppar_list_to_toppar_member_info_map_by_id( + rkcg->rkcg_group_assignment); + } else { + new_assignment_set = + rd_kafka_toppar_list_to_toppar_member_info_map(assignment); + old_assignment_set = + rd_kafka_toppar_list_to_toppar_member_info_map( + rkcg->rkcg_group_assignment); + } newly_added_set = rd_kafka_member_partitions_subtract( new_assignment_set, old_assignment_set); @@ -4830,9 +5371,20 @@ static void rd_kafka_cgrp_group_assignment_modify( RD_KAFKA_TPLIST_FOREACH(rktpar, partitions) { int idx; - idx = rd_kafka_topic_partition_list_find_idx( - rkcg->rkcg_group_assignment, rktpar->topic, - rktpar->partition); + /* Share-consumer assignment lists carry topic_id; + * key the lookup by (topic_id, partition) so a + * same-name topic recreate doesn't collide an OLD-id + * entry with a NEW-id entry that share name and + * partition. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) + idx = rd_kafka_topic_partition_list_find_idx_by_id( + rkcg->rkcg_group_assignment, + rd_kafka_topic_partition_get_topic_id(rktpar), + rktpar->partition); + else + idx = rd_kafka_topic_partition_list_find_idx( + rkcg->rkcg_group_assignment, rktpar->topic, + rktpar->partition); if (add) { rd_assert(idx == -1); @@ -5169,9 +5721,14 @@ rd_kafka_cgrp_max_poll_interval_check_tmr_cb(rd_kafka_timers_t *rkts, 1 /*lock*/); if (rkcg->rkcg_group_protocol == RD_KAFKA_GROUP_PROTOCOL_CONSUMER) { - rd_kafka_cgrp_consumer_leave(rkcg); + /* Do not send the leave heartbeat immediately: defer it until + * the assignment has been revoked (F_LEAVE_ON_UNASSIGN_DONE). + * Sending it now races the rejoin -- a fast leave reply runs + * consumer_reset, clearing F_WAIT_REJOIN before the serve loop + * acts on it, stranding the member at epoch 0. */ rkcg->rkcg_consumer_flags |= RD_KAFKA_CGRP_CONSUMER_F_WAIT_REJOIN; + rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_LEAVE_ON_UNASSIGN_DONE; rd_kafka_cgrp_consumer_expedite_next_heartbeat( rkcg, "max poll interval " @@ -5256,6 +5813,63 @@ static void rd_kafka_propagate_consumer_topic_errors( } +/** + * @brief Share-consumer entry point for end-of-metadata-cycle error + * propagation. + * + * Drains the per-cycle pending list: app-facing codes + * (TOPIC_EXCEPTION, TOPIC_AUTHORIZATION_FAILED) are emitted + * directly to the application on next poll; transient codes are + * debug-logged. + * + * TODO KIP-932: We need to throw the CONSUMER_ERR ops as a set + * of topic partitions and error code, rather than one per topic. + * + * @locality rdkafka main thread + */ +void rd_kafka_share_topic_err_propagate(rd_kafka_cgrp_t *rkcg) { + rd_kafka_t *rk = rkcg->rkcg_rk; + int i; + + if (!rkcg->rkcg_errored_topics) + return; + + for (i = 0; i < rkcg->rkcg_errored_topics->cnt; i++) { + const rd_kafka_topic_partition_t *pending_errored_tp = + &rkcg->rkcg_errored_topics->elems[i]; + + switch (pending_errored_tp->err) { + case RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION: + case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED: + rd_kafka_dbg(rk, CONSUMER | RD_KAFKA_DBG_TOPIC, + "TOPICERR", + "Subscribed topic not available: %s: %s", + pending_errored_tp->topic, + rd_kafka_err2str(pending_errored_tp->err)); + rd_kafka_consumer_err( + rkcg->rkcg_q, RD_KAFKA_NODEID_UA, + pending_errored_tp->err, 0, + pending_errored_tp->topic, NULL, + RD_KAFKA_OFFSET_INVALID, + "Subscribed topic not available: %s: %s", + pending_errored_tp->topic, + rd_kafka_err2str(pending_errored_tp->err)); + break; + + default: + rd_kafka_dbg(rk, TOPIC | RD_KAFKA_DBG_CGRP, "TOPICERR", + "Topic %s: %s", pending_errored_tp->topic, + rd_kafka_err2str(pending_errored_tp->err)); + break; + } + } + + /* Clear the per-cycle accumulator; next cycle starts fresh. */ + rd_kafka_topic_partition_list_destroy(rkcg->rkcg_errored_topics); + rkcg->rkcg_errored_topics = rd_kafka_topic_partition_list_new(0); +} + + /** * @brief Work out the topics currently subscribed to that do not * match any pattern in \p subscription. @@ -5363,23 +5977,39 @@ rd_kafka_cgrp_subscription_set(rd_kafka_cgrp_t *rkcg, if (rkcg->rkcg_subscription) { rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_SUBSCRIPTION; - if (rd_kafka_topic_partition_list_regex_cnt( - rkcg->rkcg_subscription) > 0) - rkcg->rkcg_flags |= - RD_KAFKA_CGRP_F_WILDCARD_SUBSCRIPTION; - if (rkcg->rkcg_group_protocol == - RD_KAFKA_GROUP_PROTOCOL_CONSUMER) { - rkcg->rkcg_subscription_regex = - rd_kafka_topic_partition_list_combine_regexes( - rkcg->rkcg_subscription); + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) { + /* Share consumer has no regex subscription: feed the + * full subscription list to the heartbeat as + * SubscribedTopicNames; let the broker reject any + * invalid topic names. */ rkcg->rkcg_subscription_topics = - rd_kafka_topic_partition_list_remove_regexes( + rd_kafka_topic_partition_list_copy( rkcg->rkcg_subscription); rkcg->rkcg_consumer_flags |= RD_KAFKA_CGRP_CONSUMER_F_SUBSCRIBED_ONCE | RD_KAFKA_CGRP_CONSUMER_F_SEND_NEW_SUBSCRIPTION; rd_kafka_cgrp_maybe_clear_heartbeat_failed_err(rkcg); + } else { + if (rd_kafka_topic_partition_list_regex_cnt( + rkcg->rkcg_subscription) > 0) + rkcg->rkcg_flags |= + RD_KAFKA_CGRP_F_WILDCARD_SUBSCRIPTION; + + if (rkcg->rkcg_group_protocol == + RD_KAFKA_GROUP_PROTOCOL_CONSUMER) { + rkcg->rkcg_subscription_regex = + rd_kafka_topic_partition_list_combine_regexes( + rkcg->rkcg_subscription); + rkcg->rkcg_subscription_topics = + rd_kafka_topic_partition_list_remove_regexes( + rkcg->rkcg_subscription); + rkcg->rkcg_consumer_flags |= + RD_KAFKA_CGRP_CONSUMER_F_SUBSCRIBED_ONCE | + RD_KAFKA_CGRP_CONSUMER_F_SEND_NEW_SUBSCRIPTION; + rd_kafka_cgrp_maybe_clear_heartbeat_failed_err( + rkcg); + } } } else { rkcg->rkcg_subscription_regex = NULL; @@ -5540,6 +6170,25 @@ rd_kafka_cgrp_consumer_unsubscribe(rd_kafka_cgrp_t *rkcg, * When group is already leaving we just wait that previous * leave request finishes. */ + + /* TODO KIP-932: Revisit how the fatal-error case suppresses the + * leave heartbeat and consider simplifying it. + * + * Today, after a fatal error raised from the ShareGroupHeartbeat + * response handler, the leave is suppressed implicitly: that + * handler calls revoke_all_rejoin_maybe(), and because a fatal + * error code is now set, rebalance_op_incr() eventually sets + * rkcg_generation_id = 0 within the same op. So by the time the + * application observes the fatal error (via consume_batch) and calls + * close(), the member is back in join-state INIT with generation_id == + * 0, hence HAS_JOINED below is false and no leave ShareGroupHeartbeat + * is sent. + * + * This works but is indirect and easy to break. Consider adding an + * explicit fatal-error guard here (as the Java client does) so the + * no-leave behaviour is stated directly rather than relying on the + * generation-id reset happening first. + */ if (leave_group && !rd_kafka_cgrp_consumer_will_rejoin(rkcg) && RD_KAFKA_CGRP_HAS_JOINED(rkcg) && !rd_kafka_cgrp_will_leave(rkcg)) { rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_LEAVE_ON_UNASSIGN_DONE; @@ -5705,6 +6354,8 @@ void rd_kafka_cgrp_terminate0(rd_kafka_cgrp_t *rkcg, rd_kafka_op_t *rko) { rkcg->rkcg_reply_rko ? "terminating" : "terminated"); rd_kafka_q_destroy(rkq); + /* Any share-consumer ack_batches carried on the op are + * released by the TERMINATE op destructor. */ rd_kafka_op_destroy(rko); } return; @@ -5718,6 +6369,61 @@ void rd_kafka_cgrp_terminate0(rd_kafka_cgrp_t *rkcg, rd_kafka_op_t *rko) { ~RD_KAFKA_CGRP_CONSUMER_F_WAIT_REJOIN & ~RD_KAFKA_CGRP_CONSUMER_F_WAIT_REJOIN_TO_COMPLETE; } + + /* For share groups, we have to additionally close + * sessions with all the brokers */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk) && + !rd_kafka_destroy_flags_no_consumer_close(rkcg->rkcg_rk)) { + /* TODO KIP-932: the below code is similar to + * rd_kafka_share_segregate_and_dispatch_acks() + * Check if we can reduce code duplication */ + rd_kafka_broker_t *rkb = NULL; + rkcg->rkcg_share.share_session_leave_remaining_cnt = 0; + if (rko->rko_u.terminated.ack_batches) { + rd_kafka_share_segregate_acks_by_leader( + rkcg->rkcg_rk, rko->rko_u.terminated.ack_batches); + /* Ownership of elements transferred to broker + * ack_details. Destroy the container. */ + rd_list_destroy(rko->rko_u.terminated.ack_batches); + rko->rko_u.terminated.ack_batches = NULL; + } + /* TODO KIP-932: See how we can avoid locking the handle */ + rd_kafka_rdlock(rkcg->rkcg_rk); + TAILQ_FOREACH(rkb, &rkcg->rkcg_rk->rk_brokers, rkb_link) { + if (rd_kafka_broker_or_instance_terminating(rkb) || + RD_KAFKA_BROKER_IS_LOGICAL(rkb)) + continue; + + rkcg->rkcg_share.share_session_leave_remaining_cnt++; + rd_dassert( + !rkb->rkb_pending_commit_sync.sync_ack_details); + /* For brokers that are currently processing share fetch + * requests we will enqueue the leave op when they reply + * back to the main thread */ + if (rkb->rkb_share_fetch_enqueued) { + rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "CLOSE", + "Deferring close request as " + "broker %s has in-flight request", + rd_kafka_broker_name(rkb)); + continue; + } + + rd_kafka_share_enqueue_fetch_op(rkcg->rkcg_rk, rkb, + rd_false, rd_true); + } + rd_kafka_rdunlock(rkcg->rkcg_rk); + } else if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk) && + rko->rko_u.terminated.ack_batches) { + /* NO_CONSUMER_CLOSE: we skip dispatching the final ack + * batches to brokers, but still take ownership of the + * ack_batches list so the toppar refs held inside it + * are released. */ + /* TODO KIP-932: Check if we need to invoke ack callbacks here + */ + rd_list_destroy(rko->rko_u.terminated.ack_batches); + rko->rko_u.terminated.ack_batches = NULL; + } + rkcg->rkcg_ts_terminate = rd_clock(); rkcg->rkcg_reply_rko = rko; @@ -5758,10 +6464,19 @@ void rd_kafka_cgrp_terminate0(rd_kafka_cgrp_t *rkcg, rd_kafka_op_t *rko) { */ void rd_kafka_cgrp_terminate(rd_kafka_cgrp_t *rkcg, rd_kafka_replyq_t replyq) { rd_kafka_assert(NULL, !thrd_is_current(rkcg->rkcg_rk->rk_thread)); - rd_kafka_cgrp_op(rkcg, NULL, replyq, RD_KAFKA_OP_TERMINATE, 0); + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) { + rd_list_t *ack_batches = + rd_kafka_share_build_ack_details(rkcg->rkcg_rk->rk_rkshare); + RD_MAP_CLEAR(&rkcg->rkcg_rk->rk_rkshare->rkshare_inflight_acks); + + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_TERMINATE); + rko->rko_replyq = replyq; + rko->rko_u.terminated.ack_batches = ack_batches; + rd_kafka_q_enq(rkcg->rkcg_ops, rko); + } else + rd_kafka_cgrp_op(rkcg, NULL, replyq, RD_KAFKA_OP_TERMINATE, 0); } - struct _op_timeout_offset_commit { rd_ts_t now; rd_kafka_t *rk; @@ -6133,6 +6848,16 @@ void rd_kafka_cgrp_consumer_group_heartbeat(rd_kafka_cgrp_t *rkcg, } rkcg->rkcg_expedite_heartbeat_retries++; + + if (RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) { + rd_kafka_ShareGroupHeartbeatRequest( + rkcg->rkcg_coord, rkcg->rkcg_group_id, rkcg->rkcg_member_id, + member_epoch, rkcg_client_rack, rkcg_subscription_topics, + RD_KAFKA_REPLYQ(rkcg->rkcg_ops, 0), + rd_kafka_cgrp_handle_ShareGroupHeartbeat, NULL); + return; + } + rd_kafka_ConsumerGroupHeartbeatRequest( rkcg->rkcg_coord, rkcg->rkcg_group_id, rkcg->rkcg_member_id, member_epoch, rkcg_group_instance_id, rkcg_client_rack, @@ -6283,6 +7008,8 @@ rd_kafka_cgrp_consumer_subscribe(rd_kafka_cgrp_t *rkcg, /* If member is leaving, new subscription * will be applied after the leave * ConsumerGroupHeartbeat */ + /* MILIND: how is new subscription applied after heartbeat, + * check it. */ if (!rd_kafka_cgrp_will_leave(rkcg)) rd_kafka_cgrp_consumer_apply_next_subscribe(rkcg); } else { @@ -6389,24 +7116,10 @@ static void rd_kafka_cgrp_consumer_assignment_done(rd_kafka_cgrp_t *rkcg) { rd_kafka_cgrp_consumer_incr_unassign_done(rkcg); break; - case RD_KAFKA_CGRP_JOIN_STATE_STEADY: - rd_kafka_cgrp_consumer_expedite_next_heartbeat( - rkcg, "back to steady state"); - - if (rkcg->rkcg_rebalance_rejoin) { - rkcg->rkcg_rebalance_rejoin = rd_false; - rd_kafka_cgrp_rejoin( - rkcg, - "rejoining group to redistribute " - "previously owned partitions to other " - "group members"); - break; - } - - /* FALLTHRU */ + case RD_KAFKA_CGRP_JOIN_STATE_INIT: + case RD_KAFKA_CGRP_JOIN_STATE_STEADY: { - case RD_KAFKA_CGRP_JOIN_STATE_INIT: { - rd_bool_t still_in_group = rd_true; + rd_bool_t not_in_group = rd_false; /* * There maybe a case when there are no assignments are * assigned to this consumer. In this case, while terminating @@ -6414,18 +7127,38 @@ static void rd_kafka_cgrp_consumer_assignment_done(rd_kafka_cgrp_t *rkcg) { * to intermediate state. In this scenario, last leave call is * done from here. */ - still_in_group &= !rd_kafka_cgrp_leave_maybe(rkcg); + not_in_group |= rd_kafka_cgrp_leave_maybe(rkcg); /* Check if cgrp is trying to terminate, which is safe to do * in these two states. Otherwise we'll need to wait for * the current state to decommission. */ - still_in_group &= !rd_kafka_cgrp_try_terminate(rkcg); + not_in_group |= rd_kafka_cgrp_try_terminate(rkcg); + + if (not_in_group) + break; - if (still_in_group) + if (rkcg->rkcg_join_state == RD_KAFKA_CGRP_JOIN_STATE_INIT) { rd_kafka_cgrp_consumer_expedite_next_heartbeat( - rkcg, "back to init state"); + rkcg, "Assignment Done: in init state"); + } else if (rkcg->rkcg_rebalance_rejoin) { + /* No need to expedite the HB here as it's being + * expedited in the rejoin call.*/ + rkcg->rkcg_rebalance_rejoin = rd_false; + rd_kafka_cgrp_rejoin( + rkcg, + "Assignment Done: rejoining group to redistribute " + "previously owned partitions to other " + "group members"); + } else if (rkcg->rkcg_consumer_flags & + RD_KAFKA_CGRP_CONSUMER_F_WAIT_ACK) { + rd_kafka_cgrp_consumer_expedite_next_heartbeat( + rkcg, + "Assignment Done: in steady state, waiting for " + "ack"); + } break; } + default: break; } @@ -6721,8 +7454,15 @@ static rd_kafka_op_res_t rd_kafka_cgrp_op_serve(rd_kafka_t *rk, case RD_KAFKA_OP_PARTITION_JOIN: rd_kafka_cgrp_partition_add(rkcg, rktp); - /* If terminating tell the partition to leave */ - if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_TERMINATE) + /* If terminating tell the partition to leave. + * + * Skipped for share consumers: op_fetch_stop bumps the + * version barrier, touches rktp_offset_query_tmr, and + * calls offset_store_stop — none of which should fire + * for a share-assigned rktp that never started a + * regular fetch. */ + if ((rkcg->rkcg_flags & RD_KAFKA_CGRP_F_TERMINATE) && + !RD_KAFKA_IS_SHARE_CONSUMER(rkcg->rkcg_rk)) rd_kafka_toppar_op_fetch_stop(rktp, RD_KAFKA_NO_REPLYQ); break; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_cgrp.h b/lib/librdkafka-2.15.0/src/rdkafka_cgrp.h similarity index 76% rename from lib/librdkafka-2.10.1/src/rdkafka_cgrp.h rename to lib/librdkafka-2.15.0/src/rdkafka_cgrp.h index 79a734f5fb2..028766d45fa 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_cgrp.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_cgrp.h @@ -126,6 +126,11 @@ typedef struct rd_kafka_cgrp_s { rd_kafka_q_t *rkcg_q; /* Application poll queue */ rd_kafka_q_t *rkcg_ops; /* Manager ops queue */ rd_kafka_q_t *rkcg_wait_coord_q; /* Ops awaiting coord */ + /* TODO KIP-932: rkcg_flags is written by the main thread (heartbeat + * builder/response handler) and read lock-free by the app thread on + * every poll via rd_kafka_app_polled(). TSAN flags this as a data + * race because the writer and reader are on different threads with + * no synchronization between them. */ int rkcg_flags; #define RD_KAFKA_CGRP_F_TERMINATE 0x1 /* Terminate cgrp (async) */ #define RD_KAFKA_CGRP_F_LEAVE_ON_UNASSIGN_DONE \ @@ -226,7 +231,18 @@ typedef struct rd_kafka_cgrp_s { /** The actual topics subscribed (after metadata+wildcard matching). * Sorted. */ rd_list_t *rkcg_subscribed_topics; /**< (rd_kafka_topic_info_t *) */ - /** Subscribed topics that are errored/not available. */ + /** Subscribed topics that are errored/not available. + * + * Two roles share this field, exclusively per cgrp protocol: + * + * - Classic consumer: cross-cycle dedup memo. + * rd_kafka_propagate_consumer_topic_errors wholesale-replaces + * it with the current cycle's errored set after emitting. + * + * - Share consumer (KIP-932): per-metadata-cycle (topic, err) + * accumulator. Populated by rd_kafka_share_toppar_enq_error + * and drained (then re-emptied) by + * rd_kafka_share_topic_err_propagate at cycle end. */ rd_kafka_topic_partition_list_t *rkcg_errored_topics; /** If a SUBSCRIBE op is received during a COOPERATIVE rebalance, * actioning this will be postponed until after the rebalance @@ -377,6 +393,90 @@ typedef struct rd_kafka_cgrp_s { /* Timestamp of last rebalance start */ rd_ts_t rkcg_ts_rebalance_start; + struct { + size_t last_partition_picked; /* For round-robin + * partition picking */ + rd_kafka_timer_t share_fetch_fanout_tmr; /**< Timer for + * share fetch + * fanout */ + rd_bool_t + share_fetch_more_records; /**< Global fetch-in-flight + * guard. When rd_true, a + * fetch is already in + * progress and new FANOUT + * ops should not send + * another fetch request. */ + int share_should_fetch_ops_in_flight_cnt; /**< Number of + * SHARE_FETCH ops + * with + * should_fetch=true + * currently + * in-flight. + * Incremented when + * enqueuing, + * decremented on + * reply. + * @locality main + * thread */ + int share_session_leave_remaining_cnt; /**< Number of + * session leave requests + * which are either + * pending to be sent or + * are in-flight. + * Populated when cgrp + * termination op is + * received and + * decremented whenever + * we receive a reply to + * a leave request + * @locality main + * thread */ + int64_t commit_sync_request_id_counter; /**< Global counter for + * commit_sync request + * IDs. Starts at 1, + * wraps to 1 at + * INT64_MAX. + * @locality main + * thread */ + rd_ts_t + fetch_stall_logged_last_ts; /**< Last time the + * main-thread re-trigger + * loop logged a "fetch + * stalled" condition; + * rate-limits that log. + * 0 = not currently + * logged. + * @locality main thread */ + } rkcg_share; + + /** + * Current commit_sync request state. + * Only one commit_sync can be active at a time since + * the app thread blocks until the main thread sends + * the response. + * @locality main thread + */ + struct { + int64_t id; /**< Current request ID from + * rkcg_share.commit_sync_request_id_counter. + * Set on each new commit_sync call. + * Reset to 0 when the reply is sent. + * 0 means no active request. */ + rd_ts_t abs_timeout; /**< Absolute timeout for the + * commit_sync request. */ + rd_kafka_topic_partition_list_t + *results; /**< Partition-to-error mapping. + * Populated as broker replies + * arrive. */ + int brokers_awaiting_result_cnt; /**< Number of broker + * results still + * pending. */ + rd_kafka_q_t *replyq; /**< App thread's temp queue + * to send response to. */ + rd_kafka_timer_t tmr; /**< Timeout timer. Fires at + * abs_timeout to send + * timeout response. */ + } rkcg_commit_sync_request; } rd_kafka_cgrp_t; @@ -429,6 +529,12 @@ void rd_kafka_cgrp_coord_dead(rd_kafka_cgrp_t *rkcg, const char *reason); void rd_kafka_cgrp_metadata_update_check(rd_kafka_cgrp_t *rkcg, rd_bool_t do_join); + +/** + * TODO KIP-932: Think of correct placing for this function. + */ +void rd_kafka_share_topic_err_propagate(rd_kafka_cgrp_t *rkcg); + #define rd_kafka_cgrp_get(rk) ((rk)->rk_cgrp) #define rd_kafka_cgrp_same_subscription_version(rk_cgrp, \ @@ -438,6 +544,12 @@ void rd_kafka_cgrp_metadata_update_check(rd_kafka_cgrp_t *rkcg, rd_atomic32_get(&(rk_cgrp)->rkcg_subscription_version) == \ cgrp_subscription_version)) +void rd_kafka_cgrp_partition_add(rd_kafka_cgrp_t *rkcg, + rd_kafka_toppar_t *rktp); + +void rd_kafka_cgrp_partition_del(rd_kafka_cgrp_t *rkcg, + rd_kafka_toppar_t *rktp); + void rd_kafka_cgrp_assigned_offsets_commit( rd_kafka_cgrp_t *rkcg, const rd_kafka_topic_partition_list_t *offsets, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_conf.c b/lib/librdkafka-2.15.0/src/rdkafka_conf.c similarity index 83% rename from lib/librdkafka-2.10.1/src/rdkafka_conf.c rename to lib/librdkafka-2.15.0/src/rdkafka_conf.c index e4b1c12fee8..77004ee5a4d 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_conf.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_conf.c @@ -56,6 +56,10 @@ #include #endif +#if WITH_OAUTHBEARER_OIDC +#include +#endif + struct rd_kafka_property { rd_kafka_conf_scope_t scope; const char *name; @@ -199,6 +203,15 @@ struct rd_kafka_property { "available at build time" #endif +#if WITH_OAUTHBEARER_OIDC +#define _UNSUPPORTED_HTTPS .unsupported = NULL +#else +#define _UNSUPPORTED_HTTPS \ + .unsupported = \ + "HTTPS calls depend on libcurl and OpenSSL which were not " \ + "available at build time" +#endif + #ifdef _WIN32 #define _UNSUPPORTED_WIN32_GSSAPI \ .unsupported = \ @@ -340,6 +353,20 @@ rd_kafka_conf_validate_partitioner(const struct rd_kafka_property *prop, !strcmp(val, "fnv1a_random"); } +/** + * @brief Validate that a string is non-null, non-empty, and not + * whitespace-only. + */ +static rd_bool_t rd_kafka_conf_validate_str(const char *value) { + const char *p; + if (!value || !*value) + return rd_false; + for (p = value; *p; p++) { + if (!isspace((int)*p)) + return rd_true; + } + return rd_false; +} /** * librdkafka configuration property definitions. @@ -417,7 +444,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { {_RK_GLOBAL, "message.copy.max.bytes", _RK_C_INT, _RK(msg_copy_max_size), "Maximum size for message to be copied to buffer. " "Messages larger than this will be passed by reference (zero-copy) " - "at the expense of larger iovecs.", + "at the expense of larger iovecs. " + "This property is not supported for share consumers.", 0, 1000000000, 0xffff}, {_RK_GLOBAL | _RK_MED, "receive.message.max.bytes", _RK_C_INT, _RK(recv_max_msg_size), @@ -426,7 +454,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "case of protocol hickups. " "This value must be at least `fetch.max.bytes` + 512 to allow " "for protocol overhead; the value is adjusted automatically " - "unless the configuration property is explicitly set.", + "unless the configuration property is explicitly set. For share " + "consumers, the default value is INT_MAX.", 1000, INT_MAX, 100000000}, {_RK_GLOBAL, "max.in.flight.requests.per.connection", _RK_C_INT, _RK(max_inflight), @@ -434,25 +463,39 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "This is a generic property applied to all broker communication, " "however it is primarily relevant to produce requests. " "In particular, note that other mechanisms limit the number " - "of outstanding consumer fetch request per broker to one.", + "of outstanding consumer fetch request per broker to one. " + "This property is ignored for share consumers.", 1, 1000000, 1000000}, {_RK_GLOBAL, "max.in.flight", _RK_C_ALIAS, .sdef = "max.in.flight.requests.per.connection"}, {_RK_GLOBAL, "metadata.recovery.strategy", _RK_C_S2I, _RK(metadata_recovery_strategy), "Controls how the client recovers when none of the brokers known to it " - "is available. If set to `none`, the client fails with a fatal error. " + "is available. If set to `none`, the client doesn't re-bootstrap. " "If set to `rebootstrap`, the client repeats the bootstrap process " "using `bootstrap.servers` and brokers added through " "`rd_kafka_brokers_add()`. Rebootstrapping is useful when a client " "communicates with brokers so infrequently that the set of brokers " "may change entirely before the client refreshes metadata. " "Metadata recovery is triggered when all last-known brokers appear " - "unavailable simultaneously.", + "unavailable simultaneously or the client cannot refresh metadata within " + "`metadata.recovery.rebootstrap.trigger.ms` or it's requested in a " + "metadata response.", .vdef = RD_KAFKA_METADATA_RECOVERY_STRATEGY_REBOOTSTRAP, .s2i = {{RD_KAFKA_METADATA_RECOVERY_STRATEGY_NONE, "none"}, {RD_KAFKA_METADATA_RECOVERY_STRATEGY_REBOOTSTRAP, "rebootstrap"}, {0, NULL}}}, + {_RK_GLOBAL, "metadata.recovery.rebootstrap.trigger.ms", _RK_C_INT, + _RK(metadata_recovery_rebootstrap_trigger_ms), + "If a client configured to rebootstrap using " + "`metadata.recovery.strategy=rebootstrap` " + "is unable to obtain metadata from any " + "of the brokers for this interval, " + "client repeats the bootstrap process using " + "`bootstrap.servers` configuration " + "and brokers added through " + "`rd_kafka_brokers_add()`.", + 0, INT_MAX, 300000}, {_RK_GLOBAL | _RK_DEPRECATED | _RK_HIDDEN, "metadata.request.timeout.ms", _RK_C_INT, _RK(metadata_request_timeout_ms), "Not used.", 10, 900 * 1000, 10}, @@ -503,7 +546,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { {_RK_GLOBAL, "topic.blacklist", _RK_C_PATLIST, _RK(topic_blacklist), "Topic blacklist, a comma-separated list of regular expressions " "for matching topic names that should be ignored in " - "broker metadata information as if the topics did not exist."}, + "broker metadata information as if the topics did not exist. " + "This property is not supported for share consumers."}, {_RK_GLOBAL | _RK_MED, "debug", _RK_C_S2F, _RK(debug), "A comma-separated list of debug contexts to enable. " "Detailed Producer debugging: broker,topic,msg. " @@ -593,11 +637,15 @@ static const struct rd_kafka_property rd_kafka_properties[] = { _RK(connections_max_idle_ms), "Close broker connections after the specified time of " "inactivity. " - "Disable with 0. " + "Disable with 0. For share consumers, the default value is 540000 (9 " + "mins)." "If this property is left at its default value some heuristics are " "performed to determine a suitable default value, this is currently " "limited to identifying brokers on Azure " - "(see librdkafka issue #3109 for more info).", + "(see librdkafka issue #3109 for more info). " + "Actual value can be lower, up to 2s lower, " + "only if `connections.max.idle.ms` >= 4s, " + "as jitter is added to avoid disconnecting all brokers at the same time.", 0, INT_MAX, 0}, {_RK_GLOBAL | _RK_MED | _RK_HIDDEN, "enable.sparse.connections", _RK_C_BOOL, _RK(sparse_connections), @@ -617,12 +665,14 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "The time is increased exponentially until " "`reconnect.backoff.max.ms` is reached. " "-25% to +50% jitter is applied to each reconnect backoff. " - "A value of 0 disables the backoff and reconnects immediately.", + "A value of 0 disables the backoff and reconnects immediately. For share " + "consumers, the default value is 50.", 0, 60 * 60 * 1000, 100}, {_RK_GLOBAL | _RK_MED, "reconnect.backoff.max.ms", _RK_C_INT, _RK(reconnect_backoff_max_ms), "The maximum time to wait before reconnecting to a broker " - "after the connection has been closed.", + "after the connection has been closed. For share consumers, the default " + "value is 1000.", 0, 60 * 60 * 1000, 10 * 1000}, {_RK_GLOBAL | _RK_HIGH, "statistics.interval.ms", _RK_C_INT, _RK(stats_interval_ms), @@ -772,7 +822,9 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "Further, the consumer default value is different from the Java " "consumer (true), and this property is not supported by the Java " "producer. Requires broker version >= 0.11.0.0, for older broker " - "versions only the broker configuration applies.", + "versions only the broker configuration applies. This property " + "is currently not supported for share consumers and will be enabled in " + "the General Availability (GA) release.", 0, 1, 0}, /* Security related global properties */ @@ -847,6 +899,29 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "If OpenSSL is dynamically linked the OpenSSL library's default " "path will be used (see `OPENSSLDIR` in `openssl version -a`).", _UNSUPPORTED_SSL}, + {_RK_GLOBAL, "https.ca.location", _RK_C_STR, _RK(https.ca_location), + "File or directory path to CA certificate(s) for verifying " + "HTTPS endpoints, like `sasl.oauthbearer.token.endpoint.url` used for " + "OAUTHBEARER/OIDC authentication. " + "Mutually exclusive with `https.ca.pem`. " + "Defaults: " + "On Windows the system's CA certificates are automatically looked " + "up in the Windows Root certificate store. " + "On Mac OSX this configuration defaults to `probe`. " + "It is recommended to install openssl using Homebrew, " + "to provide CA certificates. " + "On Linux install the distribution's ca-certificates package. " + "If OpenSSL is statically linked or `https.ca.location` is set to " + "`probe` a list of standard paths will be probed and the first one " + "found will be used as the default CA certificate location path. " + "If OpenSSL is dynamically linked the OpenSSL library's default " + "path will be used (see `OPENSSLDIR` in `openssl version -a`).", + _UNSUPPORTED_HTTPS}, + {_RK_GLOBAL, "https.ca.pem", _RK_C_STR, _RK(https.ca_pem), + "CA certificate string (PEM format) for verifying HTTPS endpoints. " + "Mutually exclusive with `https.ca.location`. " + "Optional: see `https.ca.location`.", + _UNSUPPORTED_HTTPS}, {_RK_GLOBAL | _RK_SENSITIVE, "ssl.ca.pem", _RK_C_STR, _RK(ssl.ca_pem), "CA certificate string (PEM format) for verifying the broker's key.", _UNSUPPORTED_SSL}, @@ -1040,7 +1115,11 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "authorization server handles. " "Only used when `sasl.oauthbearer.method` is set to \"oidc\".", _UNSUPPORTED_OIDC}, - {_RK_GLOBAL, "sasl.oauthbearer.client.secret", _RK_C_STR, + {_RK_GLOBAL, "sasl.oauthbearer.client.credentials.client.id", _RK_C_ALIAS, + .sdef = "sasl.oauthbearer.client.id"}, + {_RK_GLOBAL, "sasl.oauthbearer.client.credentials.client.secret", + _RK_C_ALIAS, .sdef = "sasl.oauthbearer.client.secret"}, + {_RK_GLOBAL | _RK_SENSITIVE, "sasl.oauthbearer.client.secret", _RK_C_STR, _RK(sasl.oauthbearer.client_secret), "Client secret only known to the application and the " "authorization server. This should be a sufficiently random string " @@ -1065,6 +1144,128 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. " "Only used when `sasl.oauthbearer.method` is set to \"oidc\".", _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.sub.claim.name", _RK_C_STR, + _RK(sasl.oauthbearer.sub_claim_name), + "JWT claim name to use as the subject (principal) when validating " + "OIDC access tokens. Must be present in the JWT payload with a " + "non-empty value. Should match the broker's " + "`sasl.oauthbearer.sub.claim.name` configuration for consistent " + "authentication. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\".", + .sdef = "sub", _UNSUPPORTED_OIDC}, + { + _RK_GLOBAL, + "sasl.oauthbearer.grant.type", + _RK_C_S2I, + _RK(sasl.oauthbearer.grant_type), + "OAuth grant type to use when communicating with the identity " + "provider.", + _UNSUPPORTED_OIDC, + .vdef = RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_CLIENT_CREDENTIALS, + .s2i = {{RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_CLIENT_CREDENTIALS, + "client_credentials"}, + {RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_JWT_BEARER, + "urn:ietf:params:oauth:grant-type:jwt-bearer"}}, + }, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.algorithm", _RK_C_S2I, + _RK(sasl.oauthbearer.assertion.algorithm), + "Algorithm the client should use to sign the assertion sent " + "to the identity provider and in the OAuth alg header in the JWT " + "assertion.", + _UNSUPPORTED_OIDC, + .vdef = RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_RS256, + .s2i = {{RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_RS256, "RS256"}, + {RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_ES256, "ES256"}}}, + {_RK_GLOBAL | _RK_SENSITIVE, "sasl.oauthbearer.assertion.private.key.file", + _RK_C_STR, _RK(sasl.oauthbearer.assertion.private_key.file), + "Path to client's private key (PEM) used for authentication " + "when using the JWT assertion.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL | _RK_SENSITIVE, + "sasl.oauthbearer.assertion.private.key.passphrase", _RK_C_STR, + _RK(sasl.oauthbearer.assertion.private_key.passphrase), + "Private key passphrase for `sasl.oauthbearer.assertion.private.key.file`" + " or `sasl.oauthbearer.assertion.private.key.pem`.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL | _RK_SENSITIVE, "sasl.oauthbearer.assertion.private.key.pem", + _RK_C_STR, _RK(sasl.oauthbearer.assertion.private_key.pem), + "Client's private key (PEM) used for authentication " + "when using the JWT assertion.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.file", _RK_C_STR, + _RK(sasl.oauthbearer.assertion.file), + "Path to the assertion file. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.claim.aud", _RK_C_STR, + _RK(sasl.oauthbearer.assertion.claim.audience), + "JWT audience claim. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.claim.exp.seconds", _RK_C_INT, + _RK(sasl.oauthbearer.assertion.claim.expiration_s), + "Assertion expiration time in seconds. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + 1, INT_MAX, 300, _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.claim.iss", _RK_C_STR, + _RK(sasl.oauthbearer.assertion.claim.issuer), + "JWT issuer claim. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.claim.jti.include", _RK_C_BOOL, + _RK(sasl.oauthbearer.assertion.claim.jti_include), + "JWT ID claim. When set to `true`, a random UUID is generated. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + 0, 1, 0, _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.claim.nbf.seconds", _RK_C_INT, + _RK(sasl.oauthbearer.assertion.claim.not_before_s), + "Assertion not before time in seconds. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + 0, INT_MAX, 60, _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.claim.sub", _RK_C_STR, + _RK(sasl.oauthbearer.assertion.claim.subject), + "JWT subject claim. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + _UNSUPPORTED_OIDC}, + {_RK_GLOBAL, "sasl.oauthbearer.assertion.jwt.template.file", _RK_C_STR, + _RK(sasl.oauthbearer.assertion.jwt_template_file), + "Path to the JWT template file. " + "Only used when `sasl.oauthbearer.method` is set to \"oidc\" and JWT " + "assertion is needed.", + _UNSUPPORTED_OIDC}, + { + _RK_GLOBAL, + "sasl.oauthbearer.metadata.authentication.type", + _RK_C_S2I, + _RK(sasl.oauthbearer.metadata_authentication.type), + "Type of metadata-based authentication to use for OAUTHBEARER/OIDC " + "`azure_imds` authenticates using the Azure IMDS endpoint. " + "Sets a default value for `sasl.oauthbearer.token.endpoint.url` if " + "missing. " + "`aws_iam` indicates AWS IAM-based authentication using " + "GetWebIdentityToken API. " + "librdkafka does not implement the aws_iam token flow natively as of " + "now. " + "Configuration values specific of chosen authentication type can be " + "passed " + "through `sasl.oauthbearer.config`.", + _UNSUPPORTED_OIDC, + .vdef = RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_NONE, + .s2i = + {{RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_NONE, + "none"}, + {RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AZURE_IMDS, + "azure_imds"}, + {RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AWS_IAM, + "aws_iam"}}, + }, /* Plugins */ {_RK_GLOBAL, "plugin.library.paths", _RK_C_STR, _RK(plugin_paths), @@ -1124,7 +1325,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "group rebalance. This should be used in combination with a larger " "`session.timeout.ms` to avoid group rebalances caused by transient " "unavailability (e.g. process restarts). " - "Requires broker version >= 2.3.0."}, + "Requires broker version >= 2.3.0. This property is not supported for " + "share consumers."}, {_RK_GLOBAL | _RK_CGRP | _RK_MED, "partition.assignment.strategy", _RK_C_STR, _RK(partition_assignment_strategy), "The name of one or more partition assignment strategies. The " @@ -1189,7 +1391,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { _RK(group_remote_assignor), "Server side assignor to use. Keep it null to make server select a " "suitable assignor for the group. " - "Available assignors: uniform or range. Default is null", + "Available assignors: uniform or range. Default is null. This property is " + "not supported for share consumers.", .sdef = NULL}, {_RK_GLOBAL | _RK_CGRP, "coordinator.query.interval.ms", _RK_C_INT, _RK(coord_query_intvl_ms), @@ -1228,7 +1431,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { _RK(auto_commit_interval_ms), "The frequency in milliseconds that the consumer offsets " "are committed (written) to offset storage. (0 = disable). " - "This setting is used by the high-level consumer.", + "This setting is used by the high-level consumer. " + "This property is ignored for share consumers.", 0, 86400 * 1000, 5 * 1000}, {_RK_GLOBAL | _RK_CONSUMER | _RK_HIGH, "enable.auto.offset.store", _RK_C_BOOL, _RK(enable_auto_offset_store), @@ -1240,7 +1444,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "queued.min.messages", _RK_C_INT, _RK(queued_min_msgs), "Minimum number of messages per topic+partition " - "librdkafka tries to maintain in the local consumer queue.", + "librdkafka tries to maintain in the local consumer queue. This property " + "is not supported for share consumers.", 1, 10000000, 100000}, {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "queued.max.messages.kbytes", _RK_C_INT, _RK(queued_max_msg_kbytes), @@ -1251,7 +1456,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "When using the legacy simple consumer or when separate " "partition queues are used this setting applies per partition. " "This value may be overshot by fetch.message.max.bytes. " - "This property has higher priority than queued.min.messages.", + "This property has higher priority than queued.min.messages. This " + "property is not supported for share consumers.", 1, INT_MAX / 1024, 0x10000 /*64MB*/}, {_RK_GLOBAL | _RK_CONSUMER, "fetch.wait.max.ms", _RK_C_INT, _RK(fetch_wait_max_ms), @@ -1266,7 +1472,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "been exceded. " "This property may need to be decreased if the queue thresholds are " "set low and the application is experiencing long (~1s) delays " - "between messages. Low values may increase CPU utilization.", + "between messages. Low values may increase CPU utilization. This property " + "is not supported for share consumers.", 0, 300 * 1000, 1000}, {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "fetch.message.max.bytes", _RK_C_INT, _RK(fetch_msg_max_bytes), @@ -1274,7 +1481,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "fetching messages from the broker. " "If the client encounters a message larger than this value " "it will gradually try to increase it until the " - "entire message can be fetched.", + "entire message can be fetched. This property is ignored for share " + "consumers.", 1, 1000000000, 1024 * 1024}, {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "max.partition.fetch.bytes", _RK_C_ALIAS, .sdef = "fetch.message.max.bytes"}, @@ -1288,19 +1496,23 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "The maximum message batch size accepted by the broker is defined " "via `message.max.bytes` (broker config) or " "`max.message.bytes` (broker topic config). " - "`fetch.max.bytes` is automatically adjusted upwards to be " + "For regular (not share) consumers, `fetch.max.bytes` is automatically " + "adjusted " + "upwards to be " "at least `message.max.bytes` (consumer config).", 0, INT_MAX - 512, 50 * 1024 * 1024 /* 50MB */}, {_RK_GLOBAL | _RK_CONSUMER, "fetch.min.bytes", _RK_C_INT, _RK(fetch_min_bytes), "Minimum number of bytes the broker responds with. " "If fetch.wait.max.ms expires the accumulated data will " - "be sent to the client regardless of this setting.", - 1, 100000000, 1}, + "be sent to the client regardless of this setting. For regular consumers, " + "this value must be in range 1..100000000", + 0, INT_MAX, 1}, {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "fetch.error.backoff.ms", _RK_C_INT, _RK(fetch_error_backoff_ms), "How long to postpone the next fetch request for a " - "topic+partition in case of a fetch error.", + "topic+partition in case of a fetch error. This property is not supported " + "for share consumers.", 0, 300 * 1000, 500}, {_RK_GLOBAL | _RK_CONSUMER | _RK_DEPRECATED, "offset.store.method", _RK_C_S2I, _RK(offset_store_method), @@ -1317,23 +1529,28 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "Controls how to read messages written transactionally: " "`read_committed` - only return transactional messages which have " "been committed. `read_uncommitted` - return all messages, even " - "transactional messages which have been aborted.", + "transactional messages which have been aborted. This property is not " + "supported for share consumers.", .vdef = RD_KAFKA_READ_COMMITTED, .s2i = {{RD_KAFKA_READ_UNCOMMITTED, "read_uncommitted"}, {RD_KAFKA_READ_COMMITTED, "read_committed"}}}, {_RK_GLOBAL | _RK_CONSUMER, "consume_cb", _RK_C_PTR, _RK(consume_cb), - "Message consume callback (set with rd_kafka_conf_set_consume_cb())"}, + "Message consume callback (set with rd_kafka_conf_set_consume_cb()). " + "This property is not supported for share consumers."}, {_RK_GLOBAL | _RK_CONSUMER, "rebalance_cb", _RK_C_PTR, _RK(rebalance_cb), "Called after consumer group has been rebalanced " - "(set with rd_kafka_conf_set_rebalance_cb())"}, + "(set with rd_kafka_conf_set_rebalance_cb()). " + "This property is not supported for share consumers."}, {_RK_GLOBAL | _RK_CONSUMER, "offset_commit_cb", _RK_C_PTR, _RK(offset_commit_cb), "Offset commit result propagation callback. " - "(set with rd_kafka_conf_set_offset_commit_cb())"}, + "(set with rd_kafka_conf_set_offset_commit_cb()). " + "This property is not supported for share consumers."}, {_RK_GLOBAL | _RK_CONSUMER, "enable.partition.eof", _RK_C_BOOL, _RK(enable_partition_eof), "Emit RD_KAFKA_RESP_ERR__PARTITION_EOF event whenever the " - "consumer reaches the end of a partition.", + "consumer reaches the end of a partition. This property is not supported " + "for share consumers.", 0, 1, 0}, {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "check.crcs", _RK_C_BOOL, _RK(check_crcs), @@ -1346,6 +1563,25 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "which indicates where this client is physically located. It " "corresponds with the broker config `broker.rack`.", .sdef = ""}, + {_RK_GLOBAL | _RK_CONSUMER, "max.poll.records", _RK_C_INT, + _RK(share.max_poll_records), + "The maximum number of records returned in a single call to " + "`rd_kafka_share_poll()`. This value is sent to the broker in the " + "ShareFetch request and therefore bounds the number of records the broker " + "acquires and returns " + "per fetch. " + "Note: this limit is currently best-effort and not strictly enforced, " + "so a poll may occasionally return more records than this value. " + "This property is only supported for share consumers.", + 1, INT_MAX, 500}, + {_RK_GLOBAL | _RK_CONSUMER | _RK_MED, "share.acknowledgement.mode", + _RK_C_STR, _RK(share.share_acknowledgement_mode), + "Acknowledgement mode for share consumers. " + "'implicit' - messages are implicitly acknowledged when the next poll " + "is called. 'explicit' - messages must be explicitly acknowledged " + "using rd_kafka_share_acknowledge*() APIs.", + .sdef = "implicit"}, + /* Global producer properties */ {_RK_GLOBAL | _RK_PRODUCER | _RK_HIGH, "transactional.id", _RK_C_STR, @@ -1403,7 +1639,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { {_RK_GLOBAL | _RK_PRODUCER | _RK_HIGH, "queue.buffering.max.messages", _RK_C_INT, _RK(queue_buffering_max_msgs), "Maximum number of messages allowed on the producer queue. " - "This queue is shared by all topics and partitions. A value of 0 disables " + "This queue is shared by all topics and partitions. A value of 0 " + "disables " "this limit.", 0, INT_MAX, 100000}, {_RK_GLOBAL | _RK_PRODUCER | _RK_HIGH, "queue.buffering.max.kbytes", @@ -1441,7 +1678,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { {_RK_GLOBAL | _RK_MED, "retry.backoff.max.ms", _RK_C_INT, _RK(retry_backoff_max_ms), - "The max backoff time in milliseconds before retrying a protocol request, " + "The max backoff time in milliseconds before retrying a protocol " + "request, " "this is the atmost backoff allowed for exponentially backed off " "requests.", 1, 300 * 1000, 1000}, @@ -1510,7 +1748,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { 0, 900000, 10}, {_RK_GLOBAL, "client.dns.lookup", _RK_C_S2I, _RK(client_dns_lookup), "Controls how the client uses DNS lookups. By default, when the lookup " - "returns multiple IP addresses for a hostname, they will all be attempted " + "returns multiple IP addresses for a hostname, they will all be " + "attempted " "for connection before the connection is considered failed. This applies " "to both bootstrap and advertised servers. If the value is set to " "`resolve_canonical_bootstrap_servers_only`, each entry will be resolved " @@ -1596,7 +1835,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "`murmur2_random` - Java Producer compatible Murmur2 hash of key " "(NULL keys are randomly partitioned. This is functionally equivalent " "to the default partitioner in the Java Producer.), " - "`fnv1a` - FNV-1a hash of key (NULL keys are mapped to single partition), " + "`fnv1a` - FNV-1a hash of key (NULL keys are mapped to single " + "partition), " "`fnv1a_random` - FNV-1a hash of key (NULL keys are randomly " "partitioned).", .sdef = "consistent_random", @@ -1658,7 +1898,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { "consumer only. When using the high-level KafkaConsumer, the " "global `auto.commit.interval.ms` property must be used instead]. " "The frequency in milliseconds that the consumer offsets " - "are committed (written) to offset storage.", + "are committed (written) to offset storage. " + "This property is ignored for share consumers.", 10, 86400 * 1000, 60 * 1000}, {_RK_TOPIC | _RK_CONSUMER | _RK_HIGH, "auto.offset.reset", _RK_C_S2I, _RKT(auto_offset_reset), @@ -1711,7 +1952,8 @@ static const struct rd_kafka_property rd_kafka_properties[] = { {_RK_TOPIC | _RK_CONSUMER, "consume.callback.max.messages", _RK_C_INT, _RKT(consume_callback_max_msgs), "Maximum number of messages to dispatch in " - "one `rd_kafka_consume_callback*()` call (0 = unlimited)", + "one `rd_kafka_consume_callback*()` call (0 = unlimited). " + "This property is not supported for share consumers.", 0, 1000000, 0}, {0, /* End */}}; @@ -2799,7 +3041,6 @@ void rd_kafka_conf_set_offset_commit_cb( } - void rd_kafka_conf_set_error_cb(rd_kafka_conf_t *conf, void (*error_cb)(rd_kafka_t *rk, int err, @@ -3773,6 +4014,213 @@ char **rd_kafka_conf_kv_split(const char **input, size_t incnt, size_t *cntp) { return out; } +/** + * @brief Get value for the config param corresponding to \p key in + * \p config, using \p pairs_sep for splitting it + * into key-value pairs and '=' for splitting keys and values. + */ +char *rd_kafka_conf_kv_get(const char *config, + const char *key, + const char pairs_sep) { + size_t i, config_pair_cnt, config_key_value_cnt; + char *ret = NULL; + char **config_key_values; + if (!config) + return NULL; + + char **config_pairs = + rd_string_split(config, pairs_sep, rd_true, &config_pair_cnt); + + config_key_values = + rd_kafka_conf_kv_split((const char **)config_pairs, config_pair_cnt, + &config_key_value_cnt); + for (i = 0; i < config_key_value_cnt / 2; i += 2) { + char *config_key = config_key_values[i]; + if (!rd_strcmp(config_key, key)) { + ret = rd_strdup(config_key_values[i + 1]); + break; + } + } + rd_free(config_key_values); + rd_free(config_pairs); + return ret; +} + +const char * +rd_kafka_conf_finalize_oauthbearer_oidc_grant_type(rd_kafka_conf_t *conf) { + switch (conf->sasl.oauthbearer.grant_type) { + case RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_CLIENT_CREDENTIALS: + if (!conf->sasl.oauthbearer.client_id) + return "`sasl.oauthbearer.client.id` is " + "mandatory when `sasl.oauthbearer.method=oidc` " + "is set"; + + if (!conf->sasl.oauthbearer.client_secret) { + return "`sasl.oauthbearer.client.secret` is " + "mandatory when `sasl.oauthbearer.method=oidc` " + "is set"; + } + break; + case RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_JWT_BEARER: + if (conf->sasl.oauthbearer.assertion.file) { + if (conf->sasl.oauthbearer.assertion.private_key.file) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion.private." + "key.file` cannot both be set"; + + if (conf->sasl.oauthbearer.assertion.private_key.pem) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion.private." + "key.pem` cannot both be set"; + + if (conf->sasl.oauthbearer.assertion.private_key + .passphrase) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion.private." + "key.passphrase` cannot both be set"; + + if (conf->sasl.oauthbearer.assertion.jwt_template_file) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "jwt.template.file` cannot both be set"; + + if (conf->sasl.oauthbearer.assertion.claim.subject) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "claim.sub` cannot both be set"; + + if (conf->sasl.oauthbearer.assertion.claim.audience) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "claim.aud` cannot both be set"; + + if (conf->sasl.oauthbearer.assertion.claim.issuer) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "claim.iss` cannot both be set"; + + if (rd_kafka_conf_is_modified( + conf, + "sasl.oauthbearer." + "assertion.claim.jti.include")) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "claim.jti.include` cannot both be set"; + + if (rd_kafka_conf_is_modified( + conf, + "sasl.oauthbearer." + "assertion.claim.exp.seconds")) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "claim.exp.seconds` cannot both be set"; + + + if (rd_kafka_conf_is_modified( + conf, + "sasl.oauthbearer." + "assertion.claim.nbf.seconds")) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.file` and " + "`sasl.oauthbearer.assertion." + "claim.nbf.seconds` cannot both be set"; + } else { + if (conf->sasl.oauthbearer.assertion.private_key.file && + conf->sasl.oauthbearer.assertion.private_key.pem) + return "Mutually exclusive properties set. " + "`sasl.oauthbearer.assertion.private." + "key.file` and " + "`sasl.oauthbearer.assertion.private." + "key.pem` cannot both be set"; + + if (!conf->sasl.oauthbearer.assertion.private_key + .file && + !conf->sasl.oauthbearer.assertion.private_key.pem) + return "`sasl.oauthbearer.assertion.private." + "key.file` or " + "`sasl.oauthbearer.assertion.private." + "key.pem` is mandatory when " + "`sasl.oauthbearer.grant.type` " + "is set to " + "`urn:ietf:params:oauth:grant-" + "type:jwt-bearer`"; + } + break; + default: + break; + } + return NULL; +} + +const char *rd_kafka_conf_finalize_oauthbearer_oidc(rd_kafka_conf_t *conf) { + const char *errstr; + if (conf->sasl.oauthbearer.method != + RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC) + return NULL; + + if (conf->sasl.enable_oauthbearer_unsecure_jwt) + return "`enable.sasl.oauthbearer.unsecure.jwt` and " + "`sasl.oauthbearer.method=oidc` are " + "mutually exclusive"; + + if (!conf->sasl.oauthbearer.token_endpoint_url) { + const char *errstr = + "`sasl.oauthbearer.token.endpoint.url` " + "is mandatory when " + "`sasl.oauthbearer.method=oidc` is set"; + if (conf->sasl.oauthbearer.metadata_authentication.type == + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AZURE_IMDS) { + char *query = rd_kafka_conf_kv_get( + conf->sasl.oauthbearer_config, "query", ','); + if (!query) + return "`sasl.oauthbearer.token.endpoint.url` " + "is mandatory for Azure IMDS " + "authentication " + "when `query` isn't set"; + rd_free(query); + } else if ( + conf->sasl.oauthbearer.metadata_authentication.type == + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AWS_IAM) { + /* AWS IAM uses region url for metadata authentication, + * so the OIDC token endpoint URL is unused. */ + } else { + return errstr; + } + } + + if (conf->sasl.oauthbearer.metadata_authentication.type == + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_NONE) { + if ((errstr = + rd_kafka_conf_finalize_oauthbearer_oidc_grant_type( + conf))) { + return errstr; + } + } + + /* Enable background thread for the builtin OIDC handler, + * unless a refresh callback has been set. */ + if (!conf->sasl.oauthbearer.token_refresh_cb) { + conf->enabled_events |= RD_KAFKA_EVENT_BACKGROUND; + conf->sasl.enable_callback_queue = 1; + } + + if (rd_kafka_conf_is_modified(conf, + "sasl.oauthbearer.sub.claim.name") && + !rd_kafka_conf_validate_str(conf->sasl.oauthbearer.sub_claim_name)) + return "`sasl.oauthbearer.sub.claim.name` must be " + "non-empty and not contain only whitespace"; + + return NULL; +} /** * @brief Verify configuration \p conf is @@ -3814,10 +4262,33 @@ const char *rd_kafka_conf_finalize(rd_kafka_type_t cltype, if (conf->ssl.ca && (conf->ssl.ca_location || conf->ssl.ca_pem)) return "`ssl.ca.location` or `ssl.ca.pem`, and memory-based " "set_ssl_cert(CERT_CA) are mutually exclusive."; + +#if WITH_OAUTHBEARER_OIDC + if (conf->https.ca_location && conf->https.ca_pem) + return "`https.ca.location` and `https.ca.pem` " + "are mutually exclusive"; + + if (conf->https.ca_location && + rd_strcmp(conf->https.ca_location, "probe") && + !rd_file_stat(conf->https.ca_location, NULL)) + return "`https.ca.location` must be " + "an existing file or directory"; + +#if !CURL_AT_LEAST_VERSION(7, 77, 0) + if (conf->https.ca_pem) + return "`https.ca.pem` requires libcurl 7.77.0 or later"; +#endif +#endif + + #ifdef __APPLE__ else if (!conf->ssl.ca && !conf->ssl.ca_location && !conf->ssl.ca_pem) /* Default ssl.ca.location to 'probe' on OSX */ rd_kafka_conf_set(conf, "ssl.ca.location", "probe", NULL, 0); + + /* Default https.ca.location to 'probe' on OSX */ + if (!conf->https.ca_location && !conf->https.ca_pem) + rd_kafka_conf_set(conf, "https.ca.location", "probe", NULL, 0); #endif #endif @@ -3829,59 +4300,213 @@ const char *rd_kafka_conf_finalize(rd_kafka_type_t cltype, "`oauthbearer_token_refresh_cb` are " "mutually exclusive"; - if (conf->sasl.enable_oauthbearer_unsecure_jwt && - conf->sasl.oauthbearer.method == - RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC) - return "`enable.sasl.oauthbearer.unsecure.jwt` and " - "`sasl.oauthbearer.method=oidc` are " - "mutually exclusive"; - - if (conf->sasl.oauthbearer.method == - RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC) { - if (!conf->sasl.oauthbearer.client_id) - return "`sasl.oauthbearer.client.id` is " - "mandatory when " - "`sasl.oauthbearer.method=oidc` is set"; - - if (!conf->sasl.oauthbearer.client_secret) { - return "`sasl.oauthbearer.client.secret` is " - "mandatory when " - "`sasl.oauthbearer.method=oidc` is set"; - } - - if (!conf->sasl.oauthbearer.token_endpoint_url) { - return "`sasl.oauthbearer.token.endpoint.url` " - "is mandatory when " - "`sasl.oauthbearer.method=oidc` is set"; - } - } - - /* Enable background thread for the builtin OIDC handler, - * unless a refresh callback has been set. */ - if (conf->sasl.oauthbearer.method == - RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC && - !conf->sasl.oauthbearer.token_refresh_cb) { - conf->enabled_events |= RD_KAFKA_EVENT_BACKGROUND; - conf->sasl.enable_callback_queue = 1; - } + errstr = rd_kafka_conf_finalize_oauthbearer_oidc(conf); + if (errstr) + return errstr; } #endif if (cltype == RD_KAFKA_CONSUMER) { + /* Share-consumer-specific config validation. + * + * The share consumer is broker-driven + * (ShareGroupHeartbeat), so a number of client-side + * properties have no meaning or must take + * library-mandatory values. The checks below reject + * any explicit set of those properties; the + * library-mandatory values are then applied at the + * end of the block, before the downstream + * group-protocol validation runs so its + * CONSUMER-branch checks also apply to share + * consumers. + */ + if (conf->share.is_share_consumer) { + if (conf->rebalance_cb) + return "`rebalance_cb` is not applicable " + "for share consumer"; + + if (conf->enabled_events & RD_KAFKA_EVENT_REBALANCE) + return "`RD_KAFKA_EVENT_REBALANCE` is not " + "applicable for share consumer"; + + if (conf->offset_commit_cb) + return "`offset_commit_cb` is not applicable " + "for share consumer"; + + if (conf->consume_cb) + return "`consume_cb` is not applicable " + "for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "enable.auto.commit")) + return "`enable.auto.commit` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, "group.protocol")) + return "`group.protocol` is not applicable " + "for share consumer"; + + if (rd_kafka_conf_is_modified(conf, "socket.max.fails")) + return "`socket.max.fails` must be 1 for " + "share consumer; changing it is not " + "allowed"; + + if (rd_kafka_conf_is_modified( + conf, "partition.assignment.strategy")) + return "`partition.assignment.strategy` is " + "not applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "group.instance.id")) + return "`group.instance.id` is not applicable " + "for share consumer"; + + if (rd_kafka_conf_is_modified(conf, "isolation.level")) + return "`isolation.level` is not applicable " + "for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "group.remote.assignor")) + return "`group.remote.assignor` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "queued.min.messages")) + return "`queued.min.messages` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified( + conf, "queued.max.messages.kbytes")) + return "`queued.max.messages.kbytes` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "fetch.queue.backoff.ms")) + return "`fetch.queue.backoff.ms` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "fetch.error.backoff.ms")) + return "`fetch.error.backoff.ms` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "enable.partition.eof")) + return "`enable.partition.eof` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, + "message.copy.max.bytes")) + return "`message.copy.max.bytes` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified(conf, "topic.blacklist")) + return "`topic.blacklist` is not " + "applicable for share consumer"; + + if (rd_kafka_conf_is_modified( + conf, "allow.auto.create.topics")) + return "allow.auto.create.topics is currently " + "not supported for share consumer " + "and will be enabled in the General " + "Availability (GA) release"; + + if (conf->topic_conf && + rd_kafka_topic_conf_is_modified( + conf->topic_conf, "auto.offset.reset")) + return "`auto.offset.reset` is not " + "applicable for share consumer"; + + if (conf->topic_conf && + rd_kafka_topic_conf_is_modified( + conf->topic_conf, + "consume.callback.max.messages")) + return "`consume.callback.max.messages` is not " + "applicable for share consumer"; + + /* The Azure heuristic (which lowers it to <4 minutes) + * still overrides this for Azure brokers. + * + * TODO KIP-932: this 9-min default is currently + * undermined because a share consumer reconnects + * immediately after an idle close while it still has + * assigned toppars (see + * rd_kafka_broker_share_consumer_serve). Fix so the + * connection stays closed until a fetch is actually + * required. */ + if (!rd_kafka_conf_is_modified( + conf, "connections.max.idle.ms")) + conf->connections_max_idle_ms = 9 * 60 * 1000; + + /* Match the Java client's reconnect-backoff defaults + * for share consumers (librdkafka's client-wide + * defaults are 100 / 10000). Only when unset, so a + * user-provided value still wins; regular consumers + * and producers keep the librdkafka defaults. */ + if (!rd_kafka_conf_is_modified(conf, + "reconnect.backoff.ms")) + conf->reconnect_backoff_ms = 50; + if (!rd_kafka_conf_is_modified( + conf, "reconnect.backoff.max.ms")) + conf->reconnect_backoff_max_ms = 1000; + + /* TODO KIP-932: Java client has no client-side receive + * cap so we are defaulting this value to INT_MAX. To be + * at par with Java, this check should not exist + * altogether for share consumers. Currently, we are + * allowing the user to set if they want to reject + * oversize msgs gracefully. Check + * rd_buf_write_ensure_contig(). + */ + if (!rd_kafka_conf_is_modified( + conf, "receive.message.max.bytes")) + conf->recv_max_msg_size = INT_MAX; + + /* TODO KIP-932: Fix max value for fetch.wait.max.ms to + * keep it at par with Java client. Remember that + * raising the cap can overflow */ + + conf->enable_auto_commit = 0; + conf->group_protocol = RD_KAFKA_GROUP_PROTOCOL_CONSUMER; + conf->socket_max_fails = 1; + } else { + if (conf->fetch_min_bytes < 1 || + conf->fetch_min_bytes > 100000000) { + /* For non-share consumers preserve librdkafka's + * historical `fetch.min.bytes` range + * of 1..100000000. The property max was raised + * to INT_MAX so share consumers can match the + * Java client; share consumers are exempt from + * this check. */ + /* TODO KIP-932: Ensure error msg is exactly + * same as returned by rd_kafka_anyconf_set_prop + */ + return "Configuration property " + "\"fetch.min.bytes\" value is outside " + "allowed range 1..100000000"; + } + } + if (conf->group_protocol == RD_KAFKA_GROUP_PROTOCOL_CLASSIC) { if (conf->max_poll_interval_ms < conf->group_session_timeout_ms) return "`max.poll.interval.ms`must be >= " "`session.timeout.ms`"; } else { +#if !HAVE_SECURE_RAND_BYTES + return "`group.protocol=consumer` requires " + "librdkafka to be built with secure random " + "number generator support"; +#endif if (rd_kafka_conf_is_modified(conf, "session.timeout.ms")) { return "`session.timeout.ms` is not supported " - "for `group.protocol=consumer`. It is " - "defined broker side"; + "for `group.protocol=consumer` or " + "share consumer. It is defined broker " + "side"; } if (rd_kafka_conf_is_modified( @@ -3895,30 +4520,40 @@ const char *rd_kafka_conf_finalize(rd_kafka_type_t cltype, if (rd_kafka_conf_is_modified(conf, "group.protocol.type")) { return "`group.protocol.type` is not supported " - "for `group.protocol=consumer`"; + "for `group.protocol=consumer` or " + "share consumer"; } if (rd_kafka_conf_is_modified( conf, "heartbeat.interval.ms")) { return "`heartbeat.interval.ms` is not " "supported " - "for `group.protocol=consumer`. It is " - "defined broker side"; + "for `group.protocol=consumer` or " + "share consumer. It is defined broker " + "side"; } } /* Automatically adjust `fetch.max.bytes` to be >= * `message.max.bytes` and <= `queued.max.message.kbytes` - * unless set by user. */ - if (rd_kafka_conf_is_modified(conf, "fetch.max.bytes")) { - if (conf->fetch_max_bytes < conf->max_msg_size) - return "`fetch.max.bytes` must be >= " - "`message.max.bytes`"; - } else { - conf->fetch_max_bytes = - RD_MAX(RD_MIN(conf->fetch_max_bytes, - conf->queued_max_msg_kbytes * 1024), - conf->max_msg_size); + * unless set by user. + */ + /* Share consumers are exempt: neither the message.max.bytes + * floor nor the queued.max.messages.kbytes clamp applies + * (both are rejected / not applicable), so fetch.max.bytes is + * used as configured. */ + if (!conf->share.is_share_consumer) { + if (rd_kafka_conf_is_modified(conf, + "fetch.max.bytes")) { + if (conf->fetch_max_bytes < conf->max_msg_size) + return "`fetch.max.bytes` must be >= " + "`message.max.bytes`"; + } else { + conf->fetch_max_bytes = RD_MAX( + RD_MIN(conf->fetch_max_bytes, + conf->queued_max_msg_kbytes * 1024), + conf->max_msg_size); + } } /* Automatically adjust 'receive.message.max.bytes' to @@ -4054,6 +4689,8 @@ const char *rd_kafka_conf_finalize(rd_kafka_type_t cltype, if (!rd_kafka_conf_is_modified(conf, "allow.auto.create.topics")) { /* Consumer: Do not allow auto create by default. * Producer: Allow auto create by default. */ + /* TODO KIP-932: Add auto topic creation support for share + * consumers */ if (cltype == RD_KAFKA_CONSUMER) conf->allow_auto_create_topics = rd_false; else if (cltype == RD_KAFKA_PRODUCER) @@ -4521,6 +5158,19 @@ int unittest_conf(void) { rd_kafka_conf_destroy(conf); + /* Verify rd_kafka_conf_validate_str */ + RD_UT_ASSERT(!rd_kafka_conf_validate_str(NULL), "NULL must be invalid"); + RD_UT_ASSERT(!rd_kafka_conf_validate_str(""), + "empty string must be invalid"); + RD_UT_ASSERT(!rd_kafka_conf_validate_str(" "), + "whitespace-only string must be invalid"); + RD_UT_ASSERT(!rd_kafka_conf_validate_str("\t\n"), + "tab/newline-only string must be invalid"); + RD_UT_ASSERT(rd_kafka_conf_validate_str("sub"), + "\"sub\" must be valid"); + RD_UT_ASSERT(rd_kafka_conf_validate_str(" sub "), + "\" sub \" must be valid"); + RD_UT_PASS(); } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_conf.h b/lib/librdkafka-2.15.0/src/rdkafka_conf.h similarity index 88% rename from lib/librdkafka-2.10.1/src/rdkafka_conf.h rename to lib/librdkafka-2.15.0/src/rdkafka_conf.h index 26b8df6f535..43ef6822688 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_conf.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_conf.h @@ -151,9 +151,30 @@ typedef enum { typedef enum { RD_KAFKA_SASL_OAUTHBEARER_METHOD_DEFAULT, - RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC + RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC, } rd_kafka_oauthbearer_method_t; +typedef enum { + RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_CLIENT_CREDENTIALS, + RD_KAFKA_SASL_OAUTHBEARER_GRANT_TYPE_JWT_BEARER, +} rd_kafka_oauthbearer_grant_type_t; + +typedef enum { + RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_RS256, + RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_ES256, +} rd_kafka_oauthbearer_assertion_algorithm_t; + +typedef enum { + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_NONE, + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AZURE_IMDS, + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_TYPE_AWS_IAM, +} rd_kafka_oauthbearer_metadata_authentication_type_t; + + +#define RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_URL_AZURE_IMDS \ + "http://169.254.169.254/metadata/identity/oauth2/token" + + typedef enum { RD_KAFKA_SSL_ENDPOINT_ID_NONE, RD_KAFKA_SSL_ENDPOINT_ID_HTTPS, /**< RFC2818 */ @@ -176,7 +197,7 @@ typedef enum { /* Increase in steps of 64 as needed. * This must be larger than sizeof(rd_kafka_[topic_]conf_t) */ -#define RD_KAFKA_CONF_PROPS_IDX_MAX (64 * 33) +#define RD_KAFKA_CONF_PROPS_IDX_MAX (64 * 36) /** * @struct rd_kafka_anyconf_t @@ -207,6 +228,7 @@ struct rd_kafka_conf_s { int msg_copy_max_size; int recv_max_msg_size; int max_inflight; + int metadata_recovery_rebootstrap_trigger_ms; int metadata_request_timeout_ms; int metadata_refresh_interval_ms; int metadata_refresh_fast_cnt; @@ -287,6 +309,11 @@ struct rd_kafka_conf_s { void *opaque); } ssl; + struct { + char *ca_location; + char *ca_pem; + } https; + struct { const struct rd_kafka_sasl_provider *provider; char *principal; @@ -316,11 +343,42 @@ struct rd_kafka_conf_s { int enable_callback_queue; struct { rd_kafka_oauthbearer_method_t method; + rd_kafka_oauthbearer_grant_type_t grant_type; char *token_endpoint_url; char *client_id; char *client_secret; char *scope; + struct { + rd_kafka_oauthbearer_assertion_algorithm_t + algorithm; + char *file; + char *jwt_template_file; + + struct { + char *subject; + char *audience; + char *issuer; + rd_bool_t jti_include; + int not_before_s; + int expiration_s; + } claim; + struct { + char *file; + char *passphrase; + char *pem; + } private_key; + + } assertion; + + struct { + rd_kafka_oauthbearer_metadata_authentication_type_t + type; + } metadata_authentication; + + char *extensions_str; + char *sub_claim_name; + rd_bool_t builtin_token_refresh_cb; /* SASL/OAUTHBEARER token refresh event callback */ void (*token_refresh_cb)(rd_kafka_t *rk, const char *oauthbearer_config, @@ -414,6 +472,18 @@ struct rd_kafka_conf_s { rd_kafkap_str_t *client_rack; + struct { + rd_bool_t is_share_consumer; /**< Is this a share consumer? + * Set directly by + * rd_kafka_share_consumer_new + * on the conf before + * rd_kafka_new is called. + * No property table entry. */ + int max_poll_records; /**< Max records returned per poll */ + char *share_acknowledgement_mode; /**< "implicit" (default) + * or "explicit" */ + } share; + /* * Producer configuration */ @@ -642,6 +712,9 @@ struct rd_kafka_topic_conf_s { char **rd_kafka_conf_kv_split(const char **input, size_t incnt, size_t *cntp); +char * +rd_kafka_conf_kv_get(const char *config, const char *key, const char pairs_sep); + void rd_kafka_anyconf_destroy(int scope, void *conf); rd_bool_t rd_kafka_conf_is_modified(const rd_kafka_conf_t *conf, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_confval.h b/lib/librdkafka-2.15.0/src/rdkafka_confval.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_confval.h rename to lib/librdkafka-2.15.0/src/rdkafka_confval.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_coord.c b/lib/librdkafka-2.15.0/src/rdkafka_coord.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_coord.c rename to lib/librdkafka-2.15.0/src/rdkafka_coord.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_coord.h b/lib/librdkafka-2.15.0/src/rdkafka_coord.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_coord.h rename to lib/librdkafka-2.15.0/src/rdkafka_coord.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_error.c b/lib/librdkafka-2.15.0/src/rdkafka_error.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_error.c rename to lib/librdkafka-2.15.0/src/rdkafka_error.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_error.h b/lib/librdkafka-2.15.0/src/rdkafka_error.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_error.h rename to lib/librdkafka-2.15.0/src/rdkafka_error.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_event.c b/lib/librdkafka-2.15.0/src/rdkafka_event.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_event.c rename to lib/librdkafka-2.15.0/src/rdkafka_event.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_event.h b/lib/librdkafka-2.15.0/src/rdkafka_event.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_event.h rename to lib/librdkafka-2.15.0/src/rdkafka_event.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_feature.c b/lib/librdkafka-2.15.0/src/rdkafka_feature.c similarity index 50% rename from lib/librdkafka-2.10.1/src/rdkafka_feature.c rename to lib/librdkafka-2.15.0/src/rdkafka_feature.c index 21779e479a3..a381d3c9306 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_feature.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_feature.c @@ -30,6 +30,7 @@ #include "rdkafka_int.h" #include "rdkafka_feature.h" +#include "rdunittest.h" #include @@ -74,8 +75,8 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_MSGVER1, .depends = { - {RD_KAFKAP_Produce, 2, 2}, - {RD_KAFKAP_Fetch, 2, 2}, + {RD_KAFKAP_Produce, 2, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_Fetch, 2, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -84,8 +85,8 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_MSGVER2, .depends = { - {RD_KAFKAP_Produce, 3, 3}, - {RD_KAFKAP_Fetch, 4, 4}, + {RD_KAFKAP_Produce, 3, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_Fetch, 4, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -97,7 +98,7 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_APIVERSION, .depends = { - {RD_KAFKAP_ApiVersion, 0, 0}, + {RD_KAFKAP_ApiVersion, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -106,22 +107,34 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_BROKER_GROUP_COORD, .depends = { - {RD_KAFKAP_FindCoordinator, 0, 0}, + {RD_KAFKAP_FindCoordinator, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, { - /* @brief >=0.9.0: Broker-based balanced consumer groups. */ + /* @brief >=0.9.0: Broker-based balanced consumer groups (classic). */ .feature = RD_KAFKA_FEATURE_BROKER_BALANCED_CONSUMER, .depends = { - {RD_KAFKAP_FindCoordinator, 0, 0}, - {RD_KAFKAP_OffsetCommit, 1, 2}, - {RD_KAFKAP_OffsetFetch, 1, 1}, - {RD_KAFKAP_JoinGroup, 0, 0}, - {RD_KAFKAP_SyncGroup, 0, 0}, - {RD_KAFKAP_Heartbeat, 0, 0}, - {RD_KAFKAP_LeaveGroup, 0, 0}, + {RD_KAFKAP_FindCoordinator, 0, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_OffsetCommit, 1, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_OffsetFetch, 1, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_JoinGroup, 0, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_SyncGroup, 0, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_Heartbeat, 0, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_LeaveGroup, 0, RD_KAFKAP_RPC_VERSION_MAX}, + {-1}, + }, + }, + { + /* @brief Broker-based balanced consumer groups (KIP 848). */ + .feature = RD_KAFKA_FEATURE_BROKER_BALANCED_CONSUMER, + .depends = + { + {RD_KAFKAP_ConsumerGroupHeartbeat, 0, + RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_OffsetCommit, 9, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_OffsetFetch, 9, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -130,8 +143,8 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_THROTTLETIME, .depends = { - {RD_KAFKAP_Produce, 1, 2}, - {RD_KAFKAP_Fetch, 1, 2}, + {RD_KAFKAP_Produce, 1, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_Fetch, 1, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, @@ -145,7 +158,18 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_SASL_GSSAPI, .depends = { - {RD_KAFKAP_JoinGroup, 0, 0}, + {RD_KAFKAP_JoinGroup, 0, RD_KAFKAP_RPC_VERSION_MAX}, + {-1}, + }, + }, + { + /* @brief >=0.10.0: SASL (GSSAPI) authentication. + * Fallback in case JoinGroup is removed along with the + * "classic" consumer group protocol. */ + .feature = RD_KAFKA_FEATURE_SASL_GSSAPI, + .depends = + { + {RD_KAFKAP_SaslHandshake, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -156,7 +180,7 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_SASL_HANDSHAKE, .depends = { - {RD_KAFKAP_SaslHandshake, 0, 0}, + {RD_KAFKAP_SaslHandshake, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -170,7 +194,7 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_LZ4, .depends = { - {RD_KAFKAP_FindCoordinator, 0, 0}, + {RD_KAFKAP_FindCoordinator, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -179,14 +203,14 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_OFFSET_TIME, .depends = { - {RD_KAFKAP_ListOffsets, 1, 1}, + {RD_KAFKAP_ListOffsets, 1, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }}, {/* @brief >=0.11.0.0: Idempotent Producer*/ .feature = RD_KAFKA_FEATURE_IDEMPOTENT_PRODUCER, .depends = { - {RD_KAFKAP_InitProducerId, 0, 0}, + {RD_KAFKAP_InitProducerId, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }}, { @@ -194,8 +218,8 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_ZSTD, .depends = { - {RD_KAFKAP_Produce, 7, 7}, - {RD_KAFKAP_Fetch, 10, 10}, + {RD_KAFKAP_Produce, 7, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_Fetch, 10, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -204,8 +228,8 @@ static const struct rd_kafka_feature_map { .feature = RD_KAFKA_FEATURE_SASL_AUTH_REQ, .depends = { - {RD_KAFKAP_SaslHandshake, 1, 1}, - {RD_KAFKAP_SaslAuthenticate, 0, 1}, + {RD_KAFKAP_SaslHandshake, 1, RD_KAFKAP_RPC_VERSION_MAX}, + {RD_KAFKAP_SaslAuthenticate, 0, RD_KAFKAP_RPC_VERSION_MAX}, {-1}, }, }, @@ -459,3 +483,416 @@ const char *rd_kafka_features2str(int features) { return ret[reti]; } + +int rd_ut_features_check_assert_features(const char *testname, + int features, + rd_bool_t all_enabled) { + int i; + for (i = 0; rd_kafka_feature_map[i].feature != 0; i++) { + rd_bool_t enabled = + (features & rd_kafka_feature_map[i].feature) > 0; + RD_UT_ASSERT( + enabled == all_enabled, + "%s: Feature \"%s\" should %sbe enabled", testname, + rd_kafka_features2str(rd_kafka_feature_map[i].feature), + all_enabled ? "" : "not "); + } + return 0; +} + +int rd_ut_features_check(void) { + rd_kafka_t rk = RD_ZERO_INIT; + rd_kafka_broker_t rkb = RD_ZERO_INIT; + rkb.rkb_rk = &rk; + int features; + int fails = 0; + + /* ApiVersions must be sorted in these structs */ + + /* No ApiVersions for testing matching no feature */ + struct rd_kafka_ApiVersion api_versions_none[] = {0}; + + /* ApiVersions removed in KIP-896 */ + struct rd_kafka_ApiVersion api_versions_kip_896[] = { + { + .ApiKey = RD_KAFKAP_Produce, + .MinVer = 3, + .MaxVer = 12, + }, + { + .ApiKey = RD_KAFKAP_Fetch, + .MinVer = 4, + .MaxVer = 17, + }, + { + .ApiKey = RD_KAFKAP_ListOffsets, + .MinVer = 1, + .MaxVer = 10, + }, + { + .ApiKey = RD_KAFKAP_Metadata, + .MinVer = 0, + .MaxVer = 13, + }, + { + .ApiKey = RD_KAFKAP_OffsetCommit, + .MinVer = 2, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_OffsetFetch, + .MinVer = 1, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_FindCoordinator, + .MinVer = 0, + .MaxVer = 6, + }, + { + .ApiKey = RD_KAFKAP_JoinGroup, + .MinVer = 2, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_Heartbeat, + .MinVer = 0, + .MaxVer = 4, + }, + { + .ApiKey = RD_KAFKAP_LeaveGroup, + .MinVer = 0, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SyncGroup, + .MinVer = 0, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SaslHandshake, + .MinVer = 0, + .MaxVer = 1, + }, + { + .ApiKey = RD_KAFKAP_ApiVersion, + .MinVer = 0, + .MaxVer = 4, + }, + { + .ApiKey = RD_KAFKAP_InitProducerId, + .MinVer = 0, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SaslAuthenticate, + .MinVer = 0, + .MaxVer = 2, + }, + }; + + /* ApiVersions to remove in next Apache Kafka version + * (see baseline in KIP-896) */ + struct rd_kafka_ApiVersion api_versions_next_apache_kafka_next[] = { + { + .ApiKey = RD_KAFKAP_Produce, + .MinVer = 7, + .MaxVer = 12, + }, + { + .ApiKey = RD_KAFKAP_Fetch, + .MinVer = 10, + .MaxVer = 17, + }, + { + .ApiKey = RD_KAFKAP_ListOffsets, + .MinVer = 4, + .MaxVer = 10, + }, + { + .ApiKey = RD_KAFKAP_Metadata, + .MinVer = 7, + .MaxVer = 13, + }, + { + .ApiKey = RD_KAFKAP_OffsetCommit, + .MinVer = 6, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_OffsetFetch, + .MinVer = 5, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_FindCoordinator, + .MinVer = 2, + .MaxVer = 6, + }, + { + .ApiKey = RD_KAFKAP_JoinGroup, + .MinVer = 3, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_Heartbeat, + .MinVer = 2, + .MaxVer = 4, + }, + { + .ApiKey = RD_KAFKAP_LeaveGroup, + .MinVer = 2, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SyncGroup, + .MinVer = 2, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SaslHandshake, + .MinVer = 1, + .MaxVer = 1, + }, + { + .ApiKey = RD_KAFKAP_ApiVersion, + .MinVer = 2, + .MaxVer = 4, + }, + { + .ApiKey = RD_KAFKAP_InitProducerId, + .MinVer = 1, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SaslAuthenticate, + .MinVer = 0, + .MaxVer = 2, + }, + }; + + /* ApiVersions don't overlap with those enabling the feature. + * At least one condition isn't matching */ + struct rd_kafka_ApiVersion api_versions_no_overlap[] = { + { + .ApiKey = RD_KAFKAP_Produce, + .MinVer = 2, + .MaxVer = 2, + }, + { + .ApiKey = RD_KAFKAP_Fetch, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_ListOffsets, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_Metadata, + .MinVer = 7, + .MaxVer = 13, + }, + { + .ApiKey = RD_KAFKAP_OffsetCommit, + .MinVer = 6, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_OffsetFetch, + .MinVer = 5, + .MaxVer = 8, + }, + { + .ApiKey = RD_KAFKAP_Heartbeat, + .MinVer = 2, + .MaxVer = 4, + }, + { + .ApiKey = RD_KAFKAP_LeaveGroup, + .MinVer = 2, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SyncGroup, + .MinVer = 2, + .MaxVer = 5, + }, + { + .ApiKey = RD_KAFKAP_SaslAuthenticate, + .MinVer = 0, + .MaxVer = 2, + }, + }; + + /* ApiVersions where the maximum version corresponds to + * the minimum version necessary for enabling all features. */ + struct rd_kafka_ApiVersion api_versions_overlap_by_one[] = { + { + .ApiKey = RD_KAFKAP_Produce, + .MinVer = 3, + .MaxVer = 7, + }, + { + .ApiKey = RD_KAFKAP_Fetch, + .MinVer = 4, + .MaxVer = 10, + }, + { + .ApiKey = RD_KAFKAP_ListOffsets, + .MinVer = 1, + .MaxVer = 1, + }, + { + .ApiKey = RD_KAFKAP_Metadata, + .MinVer = 0, + .MaxVer = 13, + }, + { + .ApiKey = RD_KAFKAP_OffsetCommit, + .MinVer = 2, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_OffsetFetch, + .MinVer = 1, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_FindCoordinator, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_JoinGroup, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_Heartbeat, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_LeaveGroup, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_SyncGroup, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_SaslHandshake, + .MinVer = 0, + .MaxVer = 1, + }, + { + .ApiKey = RD_KAFKAP_ApiVersion, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_InitProducerId, + .MinVer = 0, + .MaxVer = 0, + }, + { + .ApiKey = RD_KAFKAP_SaslAuthenticate, + .MinVer = 0, + .MaxVer = 0, + }, + }; + + /* ApiVersions where the classic way of enabling SASL_GSSAPI + * or BROKER_BALANCED_CONSUMER was removed. */ + struct rd_kafka_ApiVersion api_versions_kip_848[] = { + { + .ApiKey = RD_KAFKAP_OffsetCommit, + .MinVer = 6, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_OffsetFetch, + .MinVer = 5, + .MaxVer = 9, + }, + { + .ApiKey = RD_KAFKAP_SaslHandshake, + .MinVer = 0, + .MaxVer = 1, + }, + { + .ApiKey = RD_KAFKAP_ConsumerGroupHeartbeat, + .MinVer = 0, + .MaxVer = 1, + }, + }; + + /* With empty ApiVersions. */ + features = rd_kafka_features_check(&rkb, api_versions_none, + RD_ARRAY_SIZE(api_versions_none)); + fails += rd_ut_features_check_assert_features("no API versions", + features, rd_false); + + /* Without KIP-896 removed versions. */ + features = rd_kafka_features_check(&rkb, api_versions_kip_896, + RD_ARRAY_SIZE(api_versions_kip_896)); + fails += rd_ut_features_check_assert_features("KIP-896 API versions", + features, rd_true); + + /* Without versions to remove in next Apache Kafka major version. */ + features = rd_kafka_features_check( + &rkb, api_versions_next_apache_kafka_next, + RD_ARRAY_SIZE(api_versions_next_apache_kafka_next)); + fails += rd_ut_features_check_assert_features( + "next Apache Kafka API versions", features, rd_true); + + /* With a RPC version range non overlapping with the one necessary + * for the feature. */ + features = + rd_kafka_features_check(&rkb, api_versions_no_overlap, + RD_ARRAY_SIZE(api_versions_no_overlap)); + fails += rd_ut_features_check_assert_features( + "not overlapping API versions", features, rd_false); + + /* Only a single RPC version is overlapping with broker + * supported ones. */ + features = + rd_kafka_features_check(&rkb, api_versions_overlap_by_one, + RD_ARRAY_SIZE(api_versions_overlap_by_one)); + fails += rd_ut_features_check_assert_features( + "overlapping API versions by one version", features, rd_true); + + /* Even when removing the classic protocol and its APIs, + * these features must be enabled. */ + features = rd_kafka_features_check(&rkb, api_versions_kip_848, + RD_ARRAY_SIZE(api_versions_kip_848)); + if (!(features & RD_KAFKA_FEATURE_BROKER_BALANCED_CONSUMER) || + !(features & RD_KAFKA_FEATURE_SASL_GSSAPI)) { + fails++; + } + + + if (fails) + return fails; + + RD_UT_PASS(); +} + +/** + * @name Unit tests + * @{ + * + */ +int unittest_feature(void) { + int fails = 0; + + fails += rd_ut_features_check(); + + return fails; +} + +/**@}*/ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_feature.h b/lib/librdkafka-2.15.0/src/rdkafka_feature.h similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_feature.h rename to lib/librdkafka-2.15.0/src/rdkafka_feature.h index 9597956ee8a..fab4005aacc 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_feature.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_feature.h @@ -99,4 +99,6 @@ int rd_kafka_features_check(rd_kafka_broker_t *rkb, const char *rd_kafka_features2str(int features); +int unittest_feature(void); + #endif /* _RDKAFKA_FEATURE_H_ */ diff --git a/lib/librdkafka-2.15.0/src/rdkafka_fetcher.c b/lib/librdkafka-2.15.0/src/rdkafka_fetcher.c new file mode 100644 index 00000000000..6a29c792604 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdkafka_fetcher.c @@ -0,0 +1,3820 @@ +/* + * librdkafka - The Apache Kafka C/C++ library + * + * Copyright (c) 2022, Magnus Edenhill + * 2023, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +/** + * @name Fetcher + * + */ + +#include "rdkafka_int.h" +#include "rdkafka_offset.h" +#include "rdkafka_msgset.h" +#include "rdkafka_fetcher.h" +#include "rdkafka_request.h" +#include "rdkafka_share_acknowledgement.h" + + +/** + * Backoff the next Fetch request (due to error). + */ +static void rd_kafka_broker_fetch_backoff(rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err) { + int backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms; + rkb->rkb_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000); + rd_rkb_dbg(rkb, FETCH, "BACKOFF", "Fetch backoff for %dms: %s", + backoff_ms, rd_kafka_err2str(err)); +} + +/** + * @brief Backoff the next Fetch for specific partition + * + * @returns the absolute backoff time (the current time for no backoff). + */ +static rd_ts_t rd_kafka_toppar_fetch_backoff(rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp, + rd_kafka_resp_err_t err) { + int backoff_ms; + + /* Don't back off on reaching end of partition */ + if (err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { + rktp->rktp_ts_fetch_backoff = 0; + return rd_clock(); /* Immediate: No practical backoff */ + } + + if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) + backoff_ms = rkb->rkb_rk->rk_conf.fetch_queue_backoff_ms; + else + backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms; + + if (unlikely(!backoff_ms)) { + rktp->rktp_ts_fetch_backoff = 0; + return rd_clock(); /* Immediate: No practical backoff */ + } + + /* Certain errors that may require manual intervention should have + * a longer backoff time. */ + if (err == RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED) + backoff_ms = RD_MAX(1000, backoff_ms * 10); + + rktp->rktp_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000); + + rd_rkb_dbg(rkb, FETCH, "BACKOFF", + "%s [%" PRId32 "]: Fetch backoff for %dms%s%s", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + backoff_ms, err ? ": " : "", + err ? rd_kafka_err2str(err) : ""); + + return rktp->rktp_ts_fetch_backoff; +} + +/** + * @brief Handle preferred replica in fetch response. + * + * @locks rd_kafka_toppar_lock(rktp) and + * rd_kafka_rdlock(rk) must NOT be held. + * + * @locality broker thread + */ +static void rd_kafka_fetch_preferred_replica_handle(rd_kafka_toppar_t *rktp, + rd_kafka_buf_t *rkbuf, + rd_kafka_broker_t *rkb, + int32_t preferred_id) { + const rd_ts_t one_minute = 60 * 1000 * 1000; + const rd_ts_t five_seconds = 5 * 1000 * 1000; + rd_kafka_broker_t *preferred_rkb; + rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk; + rd_ts_t new_intvl = + rd_interval_immediate(&rktp->rktp_new_lease_intvl, one_minute, 0); + + if (new_intvl < 0) { + /* In lieu of KIP-320, the toppar is delegated back to + * the leader in the event of an offset out-of-range + * error (KIP-392 error case #4) because this scenario + * implies the preferred replica is out-of-sync. + * + * If program execution reaches here, the leader has + * relatively quickly instructed the client back to + * a preferred replica, quite possibly the same one + * as before (possibly resulting from stale metadata), + * so we back off the toppar to slow down potential + * back-and-forth. + */ + + if (rd_interval_immediate(&rktp->rktp_new_lease_log_intvl, + one_minute, 0) > 0) + rd_rkb_log(rkb, LOG_NOTICE, "FETCH", + "%.*s [%" PRId32 + "]: preferred replica " + "(%" PRId32 + ") lease changing too quickly " + "(%" PRId64 + "s < 60s): possibly due to " + "unavailable replica or stale cluster " + "state: backing off next fetch", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition, preferred_id, + (one_minute - -new_intvl) / (1000 * 1000)); + + rd_kafka_toppar_fetch_backoff(rkb, rktp, + RD_KAFKA_RESP_ERR_NO_ERROR); + } + + rd_kafka_rdlock(rk); + preferred_rkb = rd_kafka_broker_find_by_nodeid(rk, preferred_id); + rd_kafka_rdunlock(rk); + + if (preferred_rkb) { + rd_interval_reset_to_now(&rktp->rktp_lease_intvl, 0); + rd_kafka_toppar_lock(rktp); + rd_kafka_toppar_broker_update(rktp, preferred_id, preferred_rkb, + "preferred replica updated"); + rd_kafka_toppar_unlock(rktp); + rd_kafka_broker_destroy(preferred_rkb); + return; + } + + if (rd_interval_immediate(&rktp->rktp_metadata_intvl, five_seconds, 0) > + 0) { + rd_rkb_log(rkb, LOG_NOTICE, "FETCH", + "%.*s [%" PRId32 "]: preferred replica (%" PRId32 + ") " + "is unknown: refreshing metadata", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition, preferred_id); + + rd_kafka_metadata_refresh_brokers( + rktp->rktp_rkt->rkt_rk, NULL, + "preferred replica unavailable"); + } + + rd_kafka_toppar_fetch_backoff(rkb, rktp, + RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE); +} + + +/** + * @brief Handle partition-specific Fetch error. + */ +static void rd_kafka_fetch_reply_handle_partition_error( + rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp, + const struct rd_kafka_toppar_ver *tver, + rd_kafka_resp_err_t err, + int64_t HighwaterMarkOffset) { + + rd_rkb_dbg(rkb, FETCH, "FETCHERR", + "%.*s [%" PRId32 "]: Fetch failed at %s: %s", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition, + rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos), + rd_kafka_err2name(err)); + + /* Some errors should be passed to the + * application while some handled by rdkafka */ + switch (err) { + /* Errors handled by rdkafka */ + case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART: + case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER: + case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR: + case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH: + case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH: + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: + if (err == RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE) { + /* Occurs when: + * - Msg exists on broker but + * offset > HWM, or: + * - HWM is >= offset, but msg not + * yet available at that offset + * (replica is out of sync). + * - partition leader is out of sync. + * + * Handle by requesting metadata update, changing back + * to the leader, and then retrying FETCH + * (with backoff). + */ + rd_rkb_dbg(rkb, MSG, "FETCH", + "Topic %s [%" PRId32 + "]: %s not " + "available on broker %" PRId32 + " (leader %" PRId32 + "): updating metadata and retrying", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition, + rd_kafka_fetch_pos2str( + rktp->rktp_offsets.fetch_pos), + rktp->rktp_broker_id, rktp->rktp_leader_id); + } + + if (err == RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH) { + rd_rkb_dbg(rkb, MSG | RD_KAFKA_DBG_CONSUMER, "FETCH", + "Topic %s [%" PRId32 + "]: Fetch failed at %s: %s: broker %" PRId32 + "has not yet caught up on latest metadata: " + "retrying", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition, + rd_kafka_fetch_pos2str( + rktp->rktp_offsets.fetch_pos), + rd_kafka_err2str(err), rktp->rktp_broker_id); + } + + if (rktp->rktp_broker_id != rktp->rktp_leader_id) { + rd_kafka_toppar_delegate_to_leader(rktp); + } + /* Request metadata information update*/ + rd_kafka_toppar_leader_unavailable(rktp, "fetch", err); + break; + + case RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE: { + rd_kafka_fetch_pos_t err_pos; + + if (rktp->rktp_broker_id != rktp->rktp_leader_id && + rktp->rktp_offsets.fetch_pos.offset > HighwaterMarkOffset) { + rd_kafka_log(rkb->rkb_rk, LOG_WARNING, "FETCH", + "Topic %s [%" PRId32 + "]: %s " + " out of range (HighwaterMark %" PRId64 + " fetching from " + "broker %" PRId32 " (leader %" PRId32 + "): reverting to leader", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition, + rd_kafka_fetch_pos2str( + rktp->rktp_offsets.fetch_pos), + HighwaterMarkOffset, rktp->rktp_broker_id, + rktp->rktp_leader_id); + + /* Out of range error cannot be taken as definitive + * when fetching from follower. + * Revert back to the leader in lieu of KIP-320. + */ + rd_kafka_toppar_delegate_to_leader(rktp); + break; + } + + /* Application error */ + err_pos = rktp->rktp_offsets.fetch_pos; + rktp->rktp_offsets.fetch_pos.offset = RD_KAFKA_OFFSET_INVALID; + rktp->rktp_offsets.fetch_pos.leader_epoch = -1; + rd_kafka_offset_reset(rktp, rd_kafka_broker_id(rkb), err_pos, + err, + "fetch failed due to requested offset " + "not available on the broker"); + } break; + + case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED: + /* If we're not authorized to access the + * topic mark it as errored to deny + * further Fetch requests. */ + if (rktp->rktp_last_error != err) { + rd_kafka_consumer_err( + rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err, + tver->version, NULL, rktp, + rktp->rktp_offsets.fetch_pos.offset, + "Fetch from broker %" PRId32 " failed: %s", + rd_kafka_broker_id(rkb), rd_kafka_err2str(err)); + rktp->rktp_last_error = err; + } + break; + + + /* Application errors */ + case RD_KAFKA_RESP_ERR__PARTITION_EOF: + if (rkb->rkb_rk->rk_conf.enable_partition_eof) + rd_kafka_consumer_err( + rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err, + tver->version, NULL, rktp, + rktp->rktp_offsets.fetch_pos.offset, + "Fetch from broker %" PRId32 + " reached end of " + "partition at offset %" PRId64 + " (HighwaterMark %" PRId64 ")", + rd_kafka_broker_id(rkb), + rktp->rktp_offsets.fetch_pos.offset, + HighwaterMarkOffset); + break; + + case RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE: + default: /* and all other errors */ + rd_dassert(tver->version > 0); + rd_kafka_consumer_err( + rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err, + tver->version, NULL, rktp, + rktp->rktp_offsets.fetch_pos.offset, + "Fetch from broker %" PRId32 " failed at %s: %s", + rd_kafka_broker_id(rkb), + rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos), + rd_kafka_err2str(err)); + break; + } + + /* Back off the next fetch for this partition */ + rd_kafka_toppar_fetch_backoff(rkb, rktp, err); +} + +static void rd_kafkap_Fetch_reply_tags_set_topic_cnt( + rd_kafkap_Fetch_reply_tags_t *reply_tags, + int32_t TopicCnt) { + reply_tags->TopicCnt = TopicCnt; + rd_dassert(!reply_tags->Topics); + reply_tags->Topics = rd_calloc(TopicCnt, sizeof(*reply_tags->Topics)); +} + +static void +rd_kafkap_Fetch_reply_tags_set_topic(rd_kafkap_Fetch_reply_tags_t *reply_tags, + int TopicIdx, + rd_kafka_Uuid_t TopicId, + int32_t PartitionCnt) { + reply_tags->Topics[TopicIdx].TopicId = TopicId; + reply_tags->Topics[TopicIdx].PartitionCnt = PartitionCnt; + rd_dassert(!reply_tags->Topics[TopicIdx].Partitions); + reply_tags->Topics[TopicIdx].Partitions = rd_calloc( + PartitionCnt, sizeof(*reply_tags->Topics[TopicIdx].Partitions)); +} + + +static void +rd_kafkap_Fetch_reply_tags_destroy(rd_kafkap_Fetch_reply_tags_t *reply_tags) { + int i; + for (i = 0; i < reply_tags->TopicCnt; i++) { + RD_IF_FREE(reply_tags->Topics[i].Partitions, rd_free); + } + RD_IF_FREE(reply_tags->Topics, rd_free); + RD_IF_FREE(reply_tags->NodeEndpoints.NodeEndpoints, rd_free); +} + +static int rd_kafkap_Fetch_reply_tags_partition_parse( + rd_kafka_buf_t *rkbuf, + uint64_t tagtype, + uint64_t taglen, + rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags, + rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) { + switch (tagtype) { + case 1: /* CurrentLeader */ + if (rd_kafka_buf_read_CurrentLeader( + rkbuf, &PartitionTags->CurrentLeader) == -1) + goto err_parse; + TopicTags->partitions_with_leader_change_cnt++; + return 1; + default: + return 0; + } +err_parse: + return -1; +} + +static int +rd_kafkap_Fetch_reply_tags_parse(rd_kafka_buf_t *rkbuf, + uint64_t tagtype, + uint64_t taglen, + rd_kafkap_Fetch_reply_tags_t *tags) { + switch (tagtype) { + case 0: /* NodeEndpoints */ + if (rd_kafka_buf_read_NodeEndpoints(rkbuf, + &tags->NodeEndpoints) == -1) + goto err_parse; + return 1; + default: + return 0; + } +err_parse: + return -1; +} + +static void +rd_kafka_handle_Fetch_metadata_update(rd_kafka_broker_t *rkb, + rd_kafkap_Fetch_reply_tags_t *FetchTags) { + if (FetchTags->topics_with_leader_change_cnt && + FetchTags->NodeEndpoints.NodeEndpoints) { + rd_kafka_metadata_t *md = NULL; + rd_kafka_metadata_internal_t *mdi = NULL; + rd_tmpabuf_t tbuf; + int32_t nodeid; + rd_kafka_op_t *rko; + int i, changed_topic, changed_partition; + + rd_kafka_broker_lock(rkb); + nodeid = rkb->rkb_nodeid; + rd_kafka_broker_unlock(rkb); + + rd_tmpabuf_new(&tbuf, 0, rd_true /*assert on fail*/); + rd_tmpabuf_add_alloc(&tbuf, sizeof(*mdi)); + rd_kafkap_leader_discovery_tmpabuf_add_alloc_brokers( + &tbuf, &FetchTags->NodeEndpoints); + rd_kafkap_leader_discovery_tmpabuf_add_alloc_topics( + &tbuf, FetchTags->topics_with_leader_change_cnt); + for (i = 0; i < FetchTags->TopicCnt; i++) { + if (!FetchTags->Topics[i] + .partitions_with_leader_change_cnt) + continue; + rd_kafkap_leader_discovery_tmpabuf_add_alloc_topic( + &tbuf, NULL, + FetchTags->Topics[i] + .partitions_with_leader_change_cnt); + } + rd_tmpabuf_finalize(&tbuf); + + mdi = rd_tmpabuf_alloc(&tbuf, sizeof(*mdi)); + md = &mdi->metadata; + + rd_kafkap_leader_discovery_metadata_init(mdi, nodeid); + + rd_kafkap_leader_discovery_set_brokers( + &tbuf, mdi, &FetchTags->NodeEndpoints); + + rd_kafkap_leader_discovery_set_topic_cnt( + &tbuf, mdi, FetchTags->topics_with_leader_change_cnt); + + changed_topic = 0; + for (i = 0; i < FetchTags->TopicCnt; i++) { + int j; + if (!FetchTags->Topics[i] + .partitions_with_leader_change_cnt) + continue; + + rd_kafkap_leader_discovery_set_topic( + &tbuf, mdi, changed_topic, + FetchTags->Topics[i].TopicId, NULL, + FetchTags->Topics[i] + .partitions_with_leader_change_cnt); + + changed_partition = 0; + for (j = 0; j < FetchTags->Topics[i].PartitionCnt; + j++) { + if (FetchTags->Topics[i] + .Partitions[j] + .CurrentLeader.LeaderId < 0) + continue; + + rd_kafkap_Fetch_reply_tags_Partition_t + *Partition = + &FetchTags->Topics[i].Partitions[j]; + rd_kafkap_leader_discovery_set_CurrentLeader( + &tbuf, mdi, changed_topic, + changed_partition, Partition->Partition, + &Partition->CurrentLeader); + changed_partition++; + } + changed_topic++; + } + + rko = rd_kafka_op_new(RD_KAFKA_OP_METADATA_UPDATE); + rko->rko_u.metadata.md = md; + rko->rko_u.metadata.mdi = mdi; + rd_kafka_q_enq(rkb->rkb_rk->rk_ops, rko); + } +} + +/** + * @brief Per-partition FetchResponse parsing and handling. + * + * @returns an error on buffer parse failure, else RD_KAFKA_RESP_ERR_NO_ERROR. + */ +static rd_kafka_resp_err_t rd_kafka_fetch_reply_handle_partition( + rd_kafka_broker_t *rkb, + const rd_kafkap_str_t *topic, + rd_kafka_topic_t *rkt /*possibly NULL*/, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request, + int16_t ErrorCode, + rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags, + rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) { + const int log_decode_errors = LOG_ERR; + struct rd_kafka_toppar_ver *tver, tver_skel; + rd_kafka_toppar_t *rktp = NULL; + rd_kafka_aborted_txns_t *aborted_txns = NULL; + rd_slice_t save_slice; + int32_t fetch_version; + struct { + int32_t Partition; + int16_t ErrorCode; + int64_t HighwaterMarkOffset; + int64_t LastStableOffset; /* v4 */ + int64_t LogStartOffset; /* v5 */ + int32_t MessageSetSize; + int32_t PreferredReadReplica; /* v11 */ + } hdr; + rd_kafka_resp_err_t err; + int64_t end_offset; + + rd_kafka_buf_read_i32(rkbuf, &hdr.Partition); + rd_kafka_buf_read_i16(rkbuf, &hdr.ErrorCode); + if (PartitionTags) + PartitionTags->Partition = hdr.Partition; + if (ErrorCode) + hdr.ErrorCode = ErrorCode; + rd_kafka_buf_read_i64(rkbuf, &hdr.HighwaterMarkOffset); + + end_offset = hdr.HighwaterMarkOffset; + + hdr.LastStableOffset = RD_KAFKA_OFFSET_INVALID; + hdr.LogStartOffset = RD_KAFKA_OFFSET_INVALID; + if (rd_kafka_buf_ApiVersion(request) >= 4) { + int32_t AbortedTxnCnt; + int k; + rd_kafka_buf_read_i64(rkbuf, &hdr.LastStableOffset); + if (rd_kafka_buf_ApiVersion(request) >= 5) + rd_kafka_buf_read_i64(rkbuf, &hdr.LogStartOffset); + + rd_kafka_buf_read_arraycnt(rkbuf, &AbortedTxnCnt, + RD_KAFKAP_ABORTED_TRANSACTIONS_MAX); + + if (rkb->rkb_rk->rk_conf.isolation_level == + RD_KAFKA_READ_UNCOMMITTED) { + + if (unlikely(AbortedTxnCnt > 0)) { + rd_rkb_log(rkb, LOG_ERR, "FETCH", + "%.*s [%" PRId32 + "]: " + "%" PRId32 + " aborted transaction(s) " + "encountered in READ_UNCOMMITTED " + "fetch response: ignoring.", + RD_KAFKAP_STR_PR(topic), + hdr.Partition, AbortedTxnCnt); + for (k = 0; k < AbortedTxnCnt; k++) { + rd_kafka_buf_skip(rkbuf, (8 + 8)); + /* AbortedTransaction tags */ + rd_kafka_buf_skip_tags(rkbuf); + } + } + } else { + /* Older brokers may return LSO -1, + * in which case we use the HWM. */ + if (hdr.LastStableOffset >= 0) + end_offset = hdr.LastStableOffset; + + if (AbortedTxnCnt > 0) { + aborted_txns = + rd_kafka_aborted_txns_new(AbortedTxnCnt); + for (k = 0; k < AbortedTxnCnt; k++) { + int64_t PID; + int64_t FirstOffset; + rd_kafka_buf_read_i64(rkbuf, &PID); + rd_kafka_buf_read_i64(rkbuf, + &FirstOffset); + /* AbortedTransaction tags */ + rd_kafka_buf_skip_tags(rkbuf); + rd_kafka_aborted_txns_add( + aborted_txns, PID, FirstOffset); + } + rd_kafka_aborted_txns_sort(aborted_txns); + } + } + } + + if (rd_kafka_buf_ApiVersion(request) >= 11) + rd_kafka_buf_read_i32(rkbuf, &hdr.PreferredReadReplica); + else + hdr.PreferredReadReplica = -1; + /* Compact Records Array */ + rd_kafka_buf_read_arraycnt(rkbuf, &hdr.MessageSetSize, -1); + + if (unlikely(hdr.MessageSetSize < 0)) + rd_kafka_buf_parse_fail( + rkbuf, + "%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32, + RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize); + + /* Look up topic+partition */ + if (likely(rkt != NULL)) { + rd_kafka_topic_rdlock(rkt); + rktp = rd_kafka_toppar_get(rkt, hdr.Partition, + 0 /*no ua-on-miss*/); + rd_kafka_topic_rdunlock(rkt); + } + + if (unlikely(!rkt || !rktp)) { + rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC", + "Received Fetch response (error %hu) for unknown " + "topic %.*s [%" PRId32 "]: ignoring", + hdr.ErrorCode, RD_KAFKAP_STR_PR(topic), + hdr.Partition); + rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); + goto done; + } + + rd_kafka_toppar_lock(rktp); + rktp->rktp_lo_offset = hdr.LogStartOffset; + rktp->rktp_hi_offset = hdr.HighwaterMarkOffset; + /* Let the LastStable offset be the effective + * end_offset based on protocol version, that is: + * if connected to a broker that does not support + * LastStableOffset we use the HighwaterMarkOffset. */ + rktp->rktp_ls_offset = end_offset; + rd_kafka_toppar_unlock(rktp); + + if (hdr.PreferredReadReplica != -1) { + + rd_kafka_fetch_preferred_replica_handle( + rktp, rkbuf, rkb, hdr.PreferredReadReplica); + + if (unlikely(hdr.MessageSetSize != 0)) { + rd_rkb_log(rkb, LOG_WARNING, "FETCH", + "%.*s [%" PRId32 + "]: Fetch response has both preferred read " + "replica and non-zero message set size: " + "%" PRId32 ": skipping messages", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition, hdr.MessageSetSize); + rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); + } + goto done; + } + + rd_kafka_toppar_lock(rktp); + + /* Make sure toppar hasn't moved to another broker + * during the lifetime of the request. */ + if (unlikely(rktp->rktp_broker != rkb)) { + rd_kafka_toppar_unlock(rktp); + rd_rkb_dbg(rkb, MSG, "FETCH", + "%.*s [%" PRId32 + "]: partition broker has changed: " + "discarding fetch response", + RD_KAFKAP_STR_PR(topic), hdr.Partition); + rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); + goto done; + } + + fetch_version = rktp->rktp_fetch_version; + rd_kafka_toppar_unlock(rktp); + + /* Check if this Fetch is for an outdated fetch version, + * or the original rktp was removed and a new one + * created (due to partition count decreasing and + * then increasing again, which can happen in + * desynchronized clusters): if so ignore it. */ + tver_skel.rktp = rktp; + tver = rd_list_find(request->rkbuf_rktp_vers, &tver_skel, + rd_kafka_toppar_ver_cmp); + rd_kafka_assert(NULL, tver); + if (tver->rktp != rktp || tver->version < fetch_version) { + rd_rkb_dbg(rkb, MSG, "DROP", + "%s [%" PRId32 + "]: dropping outdated fetch response " + "(v%d < %d or old rktp)", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + tver->version, fetch_version); + rd_atomic64_add(&rktp->rktp_c.rx_ver_drops, 1); + rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); + goto done; + } + + rd_rkb_dbg(rkb, MSG, "FETCH", + "Topic %.*s [%" PRId32 "] MessageSet size %" PRId32 + ", error \"%s\", MaxOffset %" PRId64 ", LSO %" PRId64 + ", Ver %" PRId32 "/%" PRId32, + RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize, + rd_kafka_err2str(hdr.ErrorCode), hdr.HighwaterMarkOffset, + hdr.LastStableOffset, tver->version, fetch_version); + + /* If this is the last message of the queue, + * signal EOF back to the application. */ + if (end_offset == rktp->rktp_offsets.fetch_pos.offset && + rktp->rktp_offsets.eof_offset != end_offset) { + hdr.ErrorCode = RD_KAFKA_RESP_ERR__PARTITION_EOF; + rktp->rktp_offsets.eof_offset = end_offset; + } + + if (unlikely(hdr.ErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR)) { + /* Handle partition-level errors. */ + rd_kafka_fetch_reply_handle_partition_error( + rkb, rktp, tver, hdr.ErrorCode, hdr.HighwaterMarkOffset); + + rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize); + goto done; + } + + /* No error, clear any previous fetch error. */ + rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR; + + if (unlikely(hdr.MessageSetSize <= 0)) + goto done; + + /** + * Parse MessageSet + */ + if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice, + (size_t)hdr.MessageSetSize)) + rd_kafka_buf_check_len(rkbuf, hdr.MessageSetSize); + + /* Parse messages */ + err = rd_kafka_msgset_parse(rkbuf, request, rktp, aborted_txns, tver); + + + rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice); + /* Continue with next partition regardless of + * parse errors (which are partition-specific) */ + + /* On error: back off the fetcher for this partition */ + if (unlikely(err)) + rd_kafka_toppar_fetch_backoff(rkb, rktp, err); + + goto done; + +err_parse: + if (aborted_txns) + rd_kafka_aborted_txns_destroy(aborted_txns); + if (rktp) + rd_kafka_toppar_destroy(rktp); /*from get()*/ + return rkbuf->rkbuf_err; + +done: + if (aborted_txns) + rd_kafka_aborted_txns_destroy(aborted_txns); + if (likely(rktp != NULL)) + rd_kafka_toppar_destroy(rktp); /*from get()*/ + + if (PartitionTags) { + /* Set default LeaderId and LeaderEpoch */ + PartitionTags->CurrentLeader.LeaderId = -1; + PartitionTags->CurrentLeader.LeaderEpoch = -1; + } + rd_kafka_buf_read_tags(rkbuf, + rd_kafkap_Fetch_reply_tags_partition_parse, + TopicTags, PartitionTags); + + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +/** + * Parses and handles a Fetch reply. + * Returns 0 on success or an error code on failure. + */ +static rd_kafka_resp_err_t +rd_kafka_fetch_reply_handle(rd_kafka_broker_t *rkb, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request) { + int32_t TopicArrayCnt; + int i; + const int log_decode_errors = LOG_ERR; + rd_kafka_topic_t *rkt = NULL; + int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_kafkap_Fetch_reply_tags_t FetchTags = RD_ZERO_INIT; + rd_bool_t has_fetch_tags = rd_false; + + if (rd_kafka_buf_ApiVersion(request) >= 1) { + int32_t Throttle_Time; + rd_kafka_buf_read_i32(rkbuf, &Throttle_Time); + + rd_kafka_op_throttle_time(rkb, rkb->rkb_rk->rk_rep, + Throttle_Time); + } + + if (rd_kafka_buf_ApiVersion(request) >= 7) { + int32_t SessionId; + rd_kafka_buf_read_i16(rkbuf, &ErrorCode); + rd_kafka_buf_read_i32(rkbuf, &SessionId); + } + + rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX); + /* Verify that TopicArrayCnt seems to be in line with remaining size */ + rd_kafka_buf_check_len(rkbuf, + TopicArrayCnt * (3 /*topic min size*/ + + 4 /*PartitionArrayCnt*/ + 4 + + 2 + 8 + 4 /*inner header*/)); + + if (rd_kafka_buf_ApiVersion(request) >= 12) { + has_fetch_tags = rd_true; + rd_kafkap_Fetch_reply_tags_set_topic_cnt(&FetchTags, + TopicArrayCnt); + } + + for (i = 0; i < TopicArrayCnt; i++) { + rd_kafkap_str_t topic = RD_ZERO_INIT; + rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO; + int32_t PartitionArrayCnt; + int j; + + if (rd_kafka_buf_ApiVersion(request) > 12) { + rd_kafka_buf_read_uuid(rkbuf, &topic_id); + rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, + topic_id); + if (rkt) + topic = *rkt->rkt_topic; + } else { + rd_kafka_buf_read_str(rkbuf, &topic); + rkt = rd_kafka_topic_find0(rkb->rkb_rk, &topic); + } + + rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt, + RD_KAFKAP_PARTITIONS_MAX); + if (rd_kafka_buf_ApiVersion(request) >= 12) { + rd_kafkap_Fetch_reply_tags_set_topic( + &FetchTags, i, topic_id, PartitionArrayCnt); + } + + for (j = 0; j < PartitionArrayCnt; j++) { + if (rd_kafka_fetch_reply_handle_partition( + rkb, &topic, rkt, rkbuf, request, ErrorCode, + has_fetch_tags ? &FetchTags.Topics[i] : NULL, + has_fetch_tags + ? &FetchTags.Topics[i].Partitions[j] + : NULL)) + goto err_parse; + } + if (has_fetch_tags && + FetchTags.Topics[i].partitions_with_leader_change_cnt) { + FetchTags.topics_with_leader_change_cnt++; + } + + if (rkt) { + rd_kafka_topic_destroy0(rkt); + rkt = NULL; + } + /* Topic Tags */ + rd_kafka_buf_skip_tags(rkbuf); + } + + /* Top level tags */ + rd_kafka_buf_read_tags(rkbuf, rd_kafkap_Fetch_reply_tags_parse, + &FetchTags); + + if (rd_kafka_buf_read_remain(rkbuf) != 0) { + rd_kafka_buf_parse_fail(rkbuf, + "Remaining data after message set " + "parse: %" PRIusz " bytes", + rd_kafka_buf_read_remain(rkbuf)); + RD_NOTREACHED(); + } + rd_kafka_handle_Fetch_metadata_update(rkb, &FetchTags); + rd_kafkap_Fetch_reply_tags_destroy(&FetchTags); + + return 0; + +err_parse: + if (rkt) + rd_kafka_topic_destroy0(rkt); + rd_kafkap_Fetch_reply_tags_destroy(&FetchTags); + rd_rkb_dbg(rkb, MSG, "BADMSG", + "Bad message (Fetch v%d): " + "is broker.version.fallback incorrectly set?", + (int)request->rkbuf_reqhdr.ApiVersion); + return rkbuf->rkbuf_err; +} + +void rd_kafka_share_filter_acquired_records_and_update_ack_type( + rd_kafka_q_t *temp_fetchq, + rd_list_t *filtered_msgs, + const int64_t *FirstOffsets, + const int64_t *LastOffsets, + const int16_t *DeliveryCounts, + int32_t AcquiredRecordsArrayCnt) { + + rd_kafka_op_t *rko; + + /* Iterate through all messages in temp_fetchq and forward + * only those whose offset falls within an acquired range. + * Also set the ack type based on the op type. */ + while ((rko = rd_kafka_q_pop(temp_fetchq, RD_POLL_NOWAIT, 0)) != NULL) { + int64_t rko_offset = rd_kafka_op_get_offset(rko); + rd_bool_t in_acquired_range = rd_false; + int16_t delivery_count; + int32_t range_idx; + + /* Check if this message's offset is within any acquired range + */ + for (range_idx = 0; range_idx < AcquiredRecordsArrayCnt; + range_idx++) { + if (rko_offset >= FirstOffsets[range_idx] && + rko_offset <= LastOffsets[range_idx]) { + in_acquired_range = rd_true; + delivery_count = DeliveryCounts[range_idx]; + break; + } + } + + if (in_acquired_range) { + /* Set ack type based on op type */ + rd_kafka_msg_t *rkm = NULL; + if (unlikely(rd_kafka_op_is_ctrl_msg(rko))) { + rd_kafka_op_destroy(rko); + continue; + } + if (rko->rko_type == RD_KAFKA_OP_FETCH) { + rkm = &rko->rko_u.fetch.rkm; + rkm->rkm_u.consumer.ack_type = + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + rkm->rkm_u.consumer.delivery_count = + delivery_count; + } else if (rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) { + rkm = &rko->rko_u.err.rkm; + /* Set ack_type to RELEASE only if not already + * set by rd_kafka_share_msgset_err_ops() + * (which sets REJECT for CRC/unsupported + * errors, RELEASE for decompression errors). */ + if (rkm->rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_GAP || + rkm->rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED) + rkm->rkm_u.consumer.ack_type = + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE; + } + + /* Add to filtered messages list */ + rd_list_add(filtered_msgs, rko); + } else { + /* Discard message not in any acquired range */ + rd_kafka_op_destroy(rko); + } + } +} + + +/** + * @brief Comparator for sorting ops by offset. + * + * Note: rd_list_sort uses rd_list_cmp_trampoline which dereferences + * the pointers before calling this comparator, so we receive + * rd_kafka_op_t* directly, not rd_kafka_op_t**. + */ +static int rd_kafka_op_offset_cmp(const void *_a, const void *_b) { + const rd_kafka_op_t *a = (const rd_kafka_op_t *)_a; + const rd_kafka_op_t *b = (const rd_kafka_op_t *)_b; + int64_t off_a = rd_kafka_op_get_offset(a); + int64_t off_b = rd_kafka_op_get_offset(b); + + return (off_a > off_b) - (off_a < off_b); +} + + +/** + * @brief Check if op matches the given topic-partition. + * + * Compares the rktp (toppar) reference to ensure both topic and partition + * match, not just the partition number. + */ +static rd_bool_t rd_kafka_op_matches_toppar(const rd_kafka_op_t *rko, + const rd_kafka_toppar_t *rktp) { + return rko->rko_rktp == rktp; +} + +/** + * @brief Find the batch entry that contains the given offset. + * + * @returns The entry, or NULL if no entry contains the offset. + */ +static rd_kafka_share_ack_batch_entry_t * +rd_kafka_share_find_entry_for_offset(rd_kafka_share_ack_batches_t *batches, + int64_t offset) { + rd_kafka_share_ack_batch_entry_t *entry; + int ei; + + RD_LIST_FOREACH(entry, &batches->entries, ei) { + if (offset >= entry->start_offset && + offset <= entry->end_offset) + return entry; + } + return NULL; +} + +/** + * @brief Get ack type from a message op (FETCH or CONSUMER_ERR). + */ +static rd_kafka_share_internal_acknowledgement_type +rd_kafka_share_ack_type_from_msg_op(rd_kafka_op_t *msg_rko) { + rd_kafka_msg_t *rkm; + + if (msg_rko->rko_type == RD_KAFKA_OP_FETCH) + rkm = &msg_rko->rko_u.fetch.rkm; + else + rkm = &msg_rko->rko_u.err.rkm; + return rkm->rkm_u.consumer.ack_type; +} + +/** + * @brief Build share fetch response RKO with messages and update inflight acks. + * + * All entry types are pre-initialized to GAP in the reply handler. This + * function overwrites ACQUIRED/REJECT for each offset that has a message + * (RD_KAFKA_OP_FETCH or RD_KAFKA_OP_CONSUMER_ERR), and adds those messages + * to the response. Moves inflight_acks batches to the response RKO. + * + * @param rkb Broker handle + * @param filtered_msgs List of filtered message ops (with ack types already + * set) + * @param inflight_acks List of rd_kafka_share_ack_batches_t* (ownership + * transferred to RKO) + * + * @returns New rko containing messages and inflight_acks, or NULL if empty + */ +static rd_kafka_op_t * +rd_kafka_share_build_response_rko(rd_kafka_broker_t *rkb, + rd_list_t *filtered_msgs, + rd_list_t *inflight_acks) { + + rd_kafka_op_t *response_rko; + rd_kafka_share_ack_batches_t *batches; + int32_t msg_cnt = 0; + int32_t inflight_acks_cnt = rd_list_cnt(inflight_acks); + int pi, i; + int total_msgs = rd_list_cnt(filtered_msgs); + int msg_start_idx = 0; + int64_t total_offsets = 0; + + /* Create response rko */ + response_rko = rd_kafka_op_new(RD_KAFKA_OP_SHARE_FETCH_RESPONSE); + response_rko->rko_rk = rkb->rkb_rk; + + response_rko->rko_u.share_fetch_response.message_rkos = + rd_list_new(0, NULL); + response_rko->rko_u.share_fetch_response.inflight_acks = + rd_list_new(0, rd_kafka_share_ack_batches_destroy_free); + + /* Process each partition: set types for message offsets, add messages. + * Messages in filtered_msgs are grouped by partition in the same order + * as inflight_acks. Types are already GAP; we set ACQUIRED/REJECT only + * where we have a message. */ + RD_LIST_FOREACH(batches, inflight_acks, pi) { + + total_offsets += batches->response_acquired_offsets_count; + + rd_kafka_topic_partition_private_t *parpriv = + (rd_kafka_topic_partition_private_t *) + batches->rktpar->_private; + rd_kafka_toppar_t *rktp = parpriv->rktp; + + rd_list_t partition_msgs; + rd_list_init(&partition_msgs, 0, NULL); + + while (msg_start_idx < total_msgs) { + rd_kafka_op_t *candidate = + rd_list_elem(filtered_msgs, msg_start_idx); + if (!rd_kafka_op_matches_toppar(candidate, rktp)) + break; + rd_list_add(&partition_msgs, candidate); + msg_start_idx++; + } + + rd_list_sort(&partition_msgs, rd_kafka_op_offset_cmp); + + int partition_msg_cnt = rd_list_cnt(&partition_msgs); + for (i = 0; i < partition_msg_cnt; i++) { + rd_kafka_op_t *msg_rko = + rd_list_elem(&partition_msgs, i); + int64_t offset = rd_kafka_op_get_offset(msg_rko); + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_find_entry_for_offset(batches, + offset); + + if (unlikely(!entry)) { + rd_rkb_dbg( + rkb, FETCH, "SHAREFETCH", + "No ack entry found for offset %" PRId64 + " on %s [%" PRId32 "], skipping", + offset, batches->rktpar->topic, + batches->rktpar->partition); + rd_kafka_op_destroy(msg_rko); + continue; + } + + entry->types[offset - entry->start_offset] = + rd_kafka_share_ack_type_from_msg_op(msg_rko); + + /** + * The per message error ops (Decompression error, CRC + * error, or MagicByte Errors are tracked in the same + * list of messages as the successful messages. + * TODO KIP-932: Check if we need a new op for record + * level message errors: RD_KAFKA_OP_CONSUMER_MSG_ERR. + */ + if (msg_rko->rko_type == RD_KAFKA_OP_FETCH || + msg_rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) { + rd_list_add( + response_rko->rko_u.share_fetch_response + .message_rkos, + msg_rko); + msg_cnt++; + } + } + + rd_list_destroy(&partition_msgs); + rd_list_add( + response_rko->rko_u.share_fetch_response.inflight_acks, + rd_kafka_share_ack_batches_copy(batches)); + } + + if (msg_cnt == 0 && inflight_acks_cnt == 0) { + rd_kafka_op_destroy(response_rko); + return NULL; + } + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Built share fetch response rko with %d messages, " + "%d gaps, and %d partitions", + msg_cnt, (int32_t)(total_offsets - msg_cnt), + inflight_acks_cnt); + + return response_rko; +} + + +/** + * @brief Handle a per-partition fetch error from a ShareFetch response. + * + * @locality broker thread + */ +static void rd_kafka_share_fetch_reply_handle_partition_error( + rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp, + const rd_kafkap_str_t *topic, + int32_t partition, + rd_kafka_resp_err_t err, + const rd_kafkap_str_t *err_msg) { + + /* TODO KIP-932: Verify whether the SURFACE-only arms below + * (CORRUPT_MESSAGE, default unknown err) should also emit an + * explicit warn-level log here. They currently rely on the + * downstream OP_CONSUMER_ERR being surfaced to the app via + * consume_batch. */ + /* TODO KIP-932: write test cases for each per-partition error + * arm below once the mock cluster exposes a per-partition + * error-injection API (e.g. + * rd_kafka_mock_partition_push_share_fetch_error). Today only + * the leader-change errors are exercised via + * rd_kafka_mock_partition_set_leader; the remaining arms + * (KAFKA_STORAGE_ERROR, OFFSET_NOT_AVAILABLE, + * REPLICA_NOT_AVAILABLE, UNKNOWN_TOPIC_OR_PART, + * UNKNOWN_TOPIC_ID, INCONSISTENT_TOPIC_ID, + * TOPIC_AUTHORIZATION_FAILED, UNKNOWN_LEADER_EPOCH, + * UNKNOWN_SERVER_ERROR, CORRUPT_MESSAGE, and the default) + * have no deterministic mock trigger. */ + switch (err) { + case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER: + case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH: + case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR: + case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE: + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "%.*s [%" PRId32 + "]: ShareFetch failed: %s: %.*s: " + "triggering metadata refresh", + RD_KAFKAP_STR_PR(topic), partition, + rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg)); + rd_kafka_toppar_leader_unavailable(rktp, "sharefetch", err); + break; + + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART: + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: + case RD_KAFKA_RESP_ERR_INCONSISTENT_TOPIC_ID: + /* No per-partition recovery action; left for the next + * metadata refresh / heartbeat reconciliation to resolve. */ + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s", + RD_KAFKAP_STR_PR(topic), partition, + rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg)); + break; + + case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED: + rd_rkb_log(rkb, LOG_WARNING, "SHAREFETCH", + "%.*s [%" PRId32 "]: Not authorized to read: %.*s", + RD_KAFKAP_STR_PR(topic), partition, + RD_KAFKAP_STR_PR(err_msg)); + rd_kafka_consumer_err( + rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err, + 0, NULL, rktp, RD_KAFKA_OFFSET_INVALID, + "ShareFetch failed for %.*s [%" PRId32 "]: %.*s", + RD_KAFKAP_STR_PR(topic), partition, + RD_KAFKAP_STR_PR(err_msg)); + break; + + case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH: + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s", + RD_KAFKAP_STR_PR(topic), partition, + rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg)); + break; + + case RD_KAFKA_RESP_ERR_UNKNOWN: + rd_rkb_log(rkb, LOG_WARNING, "SHAREFETCH", + "%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s", + RD_KAFKAP_STR_PR(topic), partition, + rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg)); + break; + + case RD_KAFKA_RESP_ERR_INVALID_MSG: + rd_kafka_consumer_err( + rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err, + 0, NULL, rktp, RD_KAFKA_OFFSET_INVALID, + "Encountered corrupt message when fetching " + "topic-partition %.*s-%" PRId32 ": %.*s", + RD_KAFKAP_STR_PR(topic), partition, + RD_KAFKAP_STR_PR(err_msg)); + break; + + default: + rd_kafka_consumer_err( + rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err, + 0, NULL, rktp, RD_KAFKA_OFFSET_INVALID, + "Unexpected error code %" PRId16 + " (%s) while fetching from topic-partition " + "%.*s-%" PRId32 ": %.*s", + (int16_t)err, rd_kafka_err2name(err), + RD_KAFKAP_STR_PR(topic), partition, + RD_KAFKAP_STR_PR(err_msg)); + break; + } +} + + +/** + * @brief Parse a partition from ShareFetch response and build inflight_acks. + * + * Creates rd_kafka_share_ack_batches_t directly with per-offset tracking. + * All offsets are initialized to ACQUIRED state; types will be updated + * later when matching with actual messages (GAP for missing, REJECT for + * errors). + * + * @param rkb Broker handle + * @param topic Topic name + * @param topic_id Topic UUID + * @param rkt Topic handle (possibly NULL) + * @param rkbuf Response buffer + * @param request Request buffer + * @param temp_appq Queue to forward filtered messages + * @param batches_out Output: inflight acks batches for this partition + * + * @returns Error code or RD_KAFKA_RESP_ERR_NO_ERROR on success + * + * TODO KIP-932: Check if we can reduce the number of args in this method. + * Can we remove the topic field from the args, since it is a derived value + * from rkt and initialize a topic variable inside this function? + */ +static rd_kafka_resp_err_t rd_kafka_share_fetch_reply_handle_partition( + rd_kafka_broker_t *rkb, + const rd_kafkap_str_t *topic, + rd_kafka_Uuid_t topic_id, + rd_kafka_topic_t *rkt /*possibly NULL*/, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request, + rd_list_t *filtered_msgs, + rd_kafka_share_ack_batches_t *batches_out, + rd_list_t *request_ack_details) { + + int32_t PartitionId; + int16_t PartitionFetchErrorCode; + rd_kafkap_str_t PartitionFetchErrorStr = + RD_KAFKAP_STR_INITIALIZER_EMPTY; + int16_t AcknowledgementErrorCode; + rd_kafkap_str_t AcknowledgementErrorStr = + RD_KAFKAP_STR_INITIALIZER_EMPTY; + rd_kafkap_CurrentLeader_t CurrentLeader; + int32_t MessageSetSize; + rd_kafka_toppar_t *rktp = NULL; + struct rd_kafka_toppar_ver tver; + rd_slice_t save_slice; + const int log_decode_errors = LOG_ERR; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; + int32_t AcquiredRecordsArrayCnt; + int64_t *FirstOffsets = NULL; + int64_t *LastOffsets = NULL; + int16_t *DeliveryCounts = NULL; + rd_kafka_q_t *temp_fetchq = rd_kafka_q_new(rkb->rkb_rk); + int i; + rd_bool_t is_sorted = rd_true; + int64_t prev_end_offset = -1, size, j; + rd_kafka_share_ack_batch_entry_t *entry; + rd_kafka_topic_partition_private_t *parpriv; + char *topic_str; + + rd_kafka_buf_read_i32(rkbuf, &PartitionId); // Partition + rd_kafka_buf_read_i16(rkbuf, + &PartitionFetchErrorCode); // PartitionFetchError + rd_kafka_buf_read_str(rkbuf, &PartitionFetchErrorStr); // ErrorString + rd_kafka_buf_read_i16( + rkbuf, &AcknowledgementErrorCode); // AcknowledgementError + rd_kafka_buf_read_str( + rkbuf, &AcknowledgementErrorStr); // AcknowledgementErrorString + + /* Set the AcknowledgementErrorCode on the matching ack batch + * in request_ack_details, so the main thread can map + * per-partition ack errors to commit_sync results / + * acknowledgement callback. PartitionFetchErrorCode is for + * fetch-level errors and is handled separately. The partition + * may be present in the response but absent from + * request_ack_details (e.g. fetch-only partitions from + * toppars_to_add) — silently skip those. + * + * Conditional on _IN_PROGRESS preserves any deliberately + * pre-set err on the batch (e.g. INVALID_SHARE_SESSION_EPOCH + * from the epoch-0 strip path) — those partitions were never + * actually sent to the broker, so the broker's + * AcknowledgementErrorCode (typically NO_ERROR) is not the + * authoritative result for them. */ + if (request_ack_details) { + rd_kafka_share_ack_batches_t *batch = + rd_kafka_share_find_ack_batch_by_id(request_ack_details, + topic_id, PartitionId); + if (batch && + batch->rktpar->err == RD_KAFKA_RESP_ERR__IN_PROGRESS) + batch->rktpar->err = AcknowledgementErrorCode; + } + + if (AcknowledgementErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR) { + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "ShareFetch response for %.*s [%" PRId32 + "]: AcknowledgementError %" PRId16 " (%s)", + RD_KAFKAP_STR_PR(topic), PartitionId, + AcknowledgementErrorCode, + rd_kafka_err2str(AcknowledgementErrorCode)); + } + + rd_kafka_buf_read_CurrentLeader(rkbuf, + &CurrentLeader); // CurrentLeader + + /* Compact Records Array */ + rd_kafka_buf_read_arraycnt(rkbuf, &MessageSetSize, -1); + + if (unlikely(MessageSetSize < 0)) + rd_kafka_buf_parse_fail( + rkbuf, + "%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32, + RD_KAFKAP_STR_PR(topic), PartitionId, MessageSetSize); + + /* Look up topic+partition */ + if (likely(rkt != NULL)) { + rd_kafka_topic_rdlock(rkt); + rktp = + rd_kafka_toppar_get(rkt, PartitionId, 0 /*no ua-on-miss*/); + rd_kafka_topic_rdunlock(rkt); + } + + if (unlikely(!rkt || !rktp || PartitionFetchErrorCode)) { + int64_t tmp_first, tmp_last; + int16_t tmp_delivery; + + if (!rkt) { + /* TODO KIP-932: Recheck this branch once the + * topic-recreate session-bookkeeping bug is fixed + * (broker session may stop carrying old topic_ids + * that no longer match any local rkt). */ + rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC", + "Received Fetch response (error %hu) for " + "unknown topic %.*s [%" PRId32 "]: ignoring", + PartitionFetchErrorCode, + RD_KAFKAP_STR_PR(topic), PartitionId); + } else if (!rktp) { + /* TODO KIP-932: Verify this handling against expected + * behavior. librdkafka requires rktp to route records; + * partitions with no local rktp are silently dropped. + */ + rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC", + "Received Fetch response (error %hu) for " + "unknown partition %.*s [%" PRId32 + "]: ignoring", + PartitionFetchErrorCode, + RD_KAFKAP_STR_PR(topic), PartitionId); + } else { + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "%.*s [%" PRId32 + "]: per-partition fetch error %s", + RD_KAFKAP_STR_PR(topic), PartitionId, + rd_kafka_err2name(PartitionFetchErrorCode)); + rd_kafka_share_fetch_reply_handle_partition_error( + rkb, rktp, topic, PartitionId, + PartitionFetchErrorCode, &PartitionFetchErrorStr); + } + + rd_kafka_buf_skip(rkbuf, MessageSetSize); + /* TODO KIP-932: Consider tracking total byte size of the + * AcquiredRecords array in the protocol to allow a single + * rd_kafka_buf_skip() here instead of per-entry parsing. */ + rd_kafka_buf_read_arraycnt(rkbuf, &AcquiredRecordsArrayCnt, + -1); // AcquiredRecordsArrayCnt + for (i = 0; i < AcquiredRecordsArrayCnt; i++) { + rd_kafka_buf_read_i64(rkbuf, + &tmp_first); // FirstOffset + rd_kafka_buf_read_i64(rkbuf, &tmp_last); // LastOffset + rd_kafka_buf_read_i16(rkbuf, + &tmp_delivery); // DeliveryCount + rd_kafka_buf_skip_tags(rkbuf); // AcquiredRecords tags + } + /* No records to track for this partition. Leave + * batches_out->rktpar as NULL so the caller skips + * adding this batch to inflight_acks. */ + rd_kafka_buf_skip_tags(rkbuf); + goto done; + } + + tver.rktp = rktp; + /* There is no versioning/barrier of the records/partitions in + * share consumer case. */ + tver.version = 0; + + if (MessageSetSize > 0) { + /** + * Parse MessageSet + */ + if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice, + (size_t)MessageSetSize)) + rd_kafka_buf_check_len(rkbuf, MessageSetSize); + + /* + * Parse messages + */ + err = rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, + temp_fetchq); + + rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice); + /* Continue with next partition regardless of + * parse errors (which are partition-specific) */ + + /** + * Only inner-record parsing errors (RD_KAFKA_RESP_ERR__BAD_MSG + * / RD_KAFKA_RESP_ERR__UNDERFLOW) propagate out of + * rd_kafka_share_msgset_parse for share consumers — per-batch + * codec / CRC / unsupported-MagicByte failures are handled + * inline by emitting per-offset RELEASE/REJECT ops and are + * swallowed by the v2 reader (msgset_reader.c). When a true + * parse error bubbles up the wire data is corrupt and we can no + * longer trust the buffer position or any per-batch metadata. + * Stop processing the rest of this ShareFetch response and + * propagate the error to the caller, which aborts the whole + * response handling. + */ + if (err) + goto done; + } + + rd_kafka_buf_read_arraycnt(rkbuf, &AcquiredRecordsArrayCnt, + -1); // AcquiredRecordsArrayCnt + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "%.*s [%" PRId32 "] : AcquiredRecordsArrayCnt: %d", + RD_KAFKAP_STR_PR(topic), PartitionId, + AcquiredRecordsArrayCnt); + + /* Allocate and initialize the topic partition */ + topic_str = RD_KAFKAP_STR_DUP(topic); + batches_out->rktpar = + rd_kafka_topic_partition_new(topic_str, PartitionId); + rd_free(topic_str); + + /* Allocate and fill the private structure */ + parpriv = rd_kafka_topic_partition_private_new(); + parpriv->rktp = rd_kafka_toppar_keep(rktp); + parpriv->topic_id = topic_id; + batches_out->rktpar->_private = parpriv; + + /* Record the broker from which records were acquired so the + * acknowledgement is sent back to the same broker. The wire + * CurrentLeader hint is only set when the broker signals a + * leader change, so we use the responding broker. */ + batches_out->response_leader_id = rkb->rkb_nodeid; + batches_out->response_acquired_offsets_count = 0; + /* Pre-allocate capacity without re-initializing the list. + * batches_out->entries was already initialized by + * rd_kafka_share_ack_batches_new() with the proper + * entry destructor. */ + if (AcquiredRecordsArrayCnt > 0) + rd_list_grow(&batches_out->entries, AcquiredRecordsArrayCnt); + + if (AcquiredRecordsArrayCnt > 0) { + FirstOffsets = + rd_malloc(sizeof(*FirstOffsets) * AcquiredRecordsArrayCnt); + LastOffsets = + rd_malloc(sizeof(*LastOffsets) * AcquiredRecordsArrayCnt); + DeliveryCounts = rd_malloc(sizeof(*DeliveryCounts) * + AcquiredRecordsArrayCnt); + + /** + * TODO KIP-932: There could be an improvement where we + * segregate the records while reading the FirstOffsets and + * LastOffsets, so that we can avoid mallocs above and directly + * create the batches and fill the entries. + */ + for (i = 0; i < AcquiredRecordsArrayCnt; i++) { + rd_kafka_buf_read_i64(rkbuf, &FirstOffsets[i]); + rd_kafka_buf_read_i64(rkbuf, &LastOffsets[i]); + rd_kafka_buf_read_i16(rkbuf, &DeliveryCounts[i]); + rd_kafka_buf_skip_tags(rkbuf); + + size = LastOffsets[i] - FirstOffsets[i] + 1; + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "%.*s [%" PRId32 + "]: Acquired Records from offset %" PRId64 + " to %" PRId64 ", DeliveryCount %" PRId16, + RD_KAFKAP_STR_PR(topic), PartitionId, + FirstOffsets[i], LastOffsets[i], + DeliveryCounts[i]); + + /* Track sortedness: entries must be non-overlapping + * and in ascending order by start_offset. */ + if (is_sorted && FirstOffsets[i] <= prev_end_offset) + is_sorted = rd_false; + prev_end_offset = LastOffsets[i]; + + entry = rd_kafka_share_ack_batch_entry_new( + FirstOffsets[i], LastOffsets[i], (int32_t)size, + DeliveryCounts[i]); + + /* Initialize all offsets to GAP; build_response_rko + * will set ACQUIRED/REJECT for offsets that have a + * message. */ + for (j = 0; j < size; j++) { + entry->types[j] = + RD_KAFKA_SHARE_INTERNAL_ACK_GAP; + } + + rd_list_add(&batches_out->entries, entry); + batches_out->response_acquired_offsets_count += + (int32_t)size; + } + + /* Mark as sorted to enable binary search in rd_list_find(). */ + if (is_sorted) + batches_out->entries.rl_flags |= RD_LIST_F_SORTED; + + /* Filter and forward messages in acquired ranges */ + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, AcquiredRecordsArrayCnt); + } else { + /* No acquired ranges: drop everything */ + rd_kafka_op_t *rko; + while ((rko = rd_kafka_q_pop(temp_fetchq, RD_POLL_NOWAIT, 0))) + rd_kafka_op_destroy(rko); + } + + rd_kafka_buf_skip_tags(rkbuf); // Partition tags + + goto done; + +err_parse: + err = rkbuf->rkbuf_err; + +done: + RD_IF_FREE(FirstOffsets, rd_free); + RD_IF_FREE(LastOffsets, rd_free); + RD_IF_FREE(DeliveryCounts, rd_free); + RD_IF_FREE(temp_fetchq, rd_kafka_q_destroy_owner); + if (rktp) + rd_kafka_toppar_destroy(rktp); /* from toppar_get() */ + return err; +} + + +/** + * Parses and handles a ShareFetch reply. + * Returns 0 on success or an error code on failure. + * + * TODO KIP-932: Change return type to proper error with message. See + * `rd_kafka_error_t *`. + */ +static rd_kafka_resp_err_t +rd_kafka_share_fetch_reply_handle(rd_kafka_broker_t *rkb, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request, + rd_kafka_op_t **response_rko_out) { + int32_t TopicArrayCnt; + int i; + const int log_decode_errors = LOG_ERR; + rd_kafka_topic_t *rkt = NULL; + int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_kafkap_str_t ErrorStr = RD_KAFKAP_STR_INITIALIZER_EMPTY; + int32_t AcquisitionLockTimeoutMs = 0; + rd_kafkap_NodeEndpoints_t NodeEndpoints; + NodeEndpoints.NodeEndpoints = NULL; + NodeEndpoints.NodeEndpointCnt = 0; + rd_list_t *filtered_msgs = NULL; + rd_list_t *inflight_acks = NULL; + rd_kafka_op_t *rko_orig = request->rkbuf_opaque; + rd_kafka_op_t *response_rko = NULL; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_kafka_op_t *fetch_size_rko; + int fetch_size_idx; + int64_t total_fetch_size_bytes = 0; + + rd_kafka_buf_read_throttle_time(rkbuf); + + rd_kafka_buf_read_i16(rkbuf, &ErrorCode); + rd_kafka_buf_read_str(rkbuf, &ErrorStr); + + if (ErrorCode) { + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "ShareFetch response error %s: '%.*s'", + rd_kafka_err2name(ErrorCode), + RD_KAFKAP_STR_PR(&ErrorStr)); + return ErrorCode; + } + + /* Count this successful ShareFetch response for + * consumer.share.fetch.manager.fetch.{total,rate}. + * Fires only after top level error handling. + * + * TODO KIP-932: Revisit this sampling before GA. It fires here, + * before the message set is parsed (rd_kafka_share_msgset_parse in + * the partition loop below). If that parse fails with + * RD_KAFKA_RESP_ERR__BAD_MSG / __UNDERFLOW the caller drops the + * whole response, yet fetch.{total,rate} and fetch.latency have + * already been counted here. */ + rd_atomic64_add(&rkb->rkb_rk->rk_telemetry.share_fetch_total, 1); + + /* rkbuf_ts_sent is already the request RTT here */ + rd_avg_add( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency, + request->rkbuf_ts_sent); + + rd_kafka_buf_read_i32(rkbuf, &AcquisitionLockTimeoutMs); + + rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX); + + filtered_msgs = rd_list_new(0, NULL); + inflight_acks = rd_list_new(0, rd_kafka_share_ack_batches_destroy_free); + + for (i = 0; i < TopicArrayCnt; i++) { + rd_kafkap_str_t topic = RD_ZERO_INIT; + rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO; + int32_t PartitionArrayCnt; + int j; + + rd_kafka_buf_read_uuid(rkbuf, &topic_id); + rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, topic_id); + if (rkt) + topic = *rkt->rkt_topic; + + rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt, + RD_KAFKAP_PARTITIONS_MAX); + + for (j = 0; j < PartitionArrayCnt; j++) { + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new_empty(); + + err = rd_kafka_share_fetch_reply_handle_partition( + rkb, &topic, topic_id, rkt, rkbuf, request, + filtered_msgs, batches, + rko_orig->rko_u.share_fetch.ack_details); + + /* + * Only inner-record parsing errors + * (RD_KAFKA_RESP_ERR__BAD_MSG / + * RD_KAFKA_RESP_ERR__UNDERFLOW) reach this point — + * per-batch CRC / decompression / MagicByte failures + * are handled inline inside + * rd_kafka_share_msgset_parse by emitting per-offset + * RELEASE/REJECT ops and are swallowed, and + * per-partition broker errors are handled by + * rd_kafka_share_fetch_reply_handle_partition_error + * before MessageSet parsing begins. A parsing error + * means the wire data is corrupt and the buffer + * position can no longer be trusted for subsequent + * partitions, so abort the whole response. + */ + if (err) { + rd_kafka_share_ack_batches_destroy(batches); + goto done; + } + + /* Skip unknown topics - don't add to inflight_acks */ + if (batches->rktpar == NULL) { + rd_kafka_share_ack_batches_destroy(batches); + continue; + } + + if (rd_list_cnt(&batches->entries) > 0) { + rd_list_add(inflight_acks, batches); + } else { + rd_kafka_share_ack_batches_destroy(batches); + } + } + + if (rkt) { + rd_kafka_topic_destroy0(rkt); + rkt = NULL; + } + /* Topic Tags */ + rd_kafka_buf_skip_tags(rkbuf); + } + + rd_kafka_buf_read_NodeEndpoints(rkbuf, &NodeEndpoints); + + /* Sum per-record wire bytes across acquired records for + * consumer.share.fetch.manager.fetch.size.{avg,max} and + * .bytes.consumed.{total,rate}. */ + RD_LIST_FOREACH(fetch_size_rko, filtered_msgs, fetch_size_idx) { + if (fetch_size_rko->rko_type == RD_KAFKA_OP_FETCH) { + rd_kafka_msg_t *rkm = &fetch_size_rko->rko_u.fetch.rkm; + total_fetch_size_bytes += + (int64_t)rkm->rkm_u.consumer.wire_size; + } + } + /* Update bytes per fetch metrics */ + rd_avg_add(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size, + total_fetch_size_bytes); + + /* Update total bytes consumed */ + rd_atomic64_add(&rkb->rkb_rk->rk_telemetry.share_bytes_consumed_total, + total_fetch_size_bytes); + + /* Build response rko with messages and inflight_acks */ + response_rko = rd_kafka_share_build_response_rko(rkb, filtered_msgs, + inflight_acks); + + rd_list_destroy(inflight_acks); + inflight_acks = NULL; + rd_list_destroy(filtered_msgs); + filtered_msgs = NULL; + + /* Top level tags */ + rd_kafka_buf_skip_tags(rkbuf); + + /* Return response_rko to the caller instead of enqueueing here. + * The caller enqueues it on rkcg_q AFTER sending the reply to + * rk_ops, ensuring the main thread processes the reply (and + * resets share_fetch_more_records) before the app thread can + * wake up, poll again, and enqueue a new FANOUT. */ + *response_rko_out = response_rko; + + /* Signal main thread whether records were fetched */ + if (rko_orig) + rko_orig->rko_u.share_fetch.records_fetched = + response_rko ? rd_true : rd_false; + + RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free); + RD_IF_FREE(rkt, rd_kafka_topic_destroy0); + return RD_KAFKA_RESP_ERR_NO_ERROR; + +err_parse: + err = rkbuf->rkbuf_err; + +done: + /* Lists are NULLed on the success path after their inline destroy; + * here they're only non-NULL on the error path. filtered_msgs uses a + * NULL destructor, so any ops accumulated from earlier partitions + * before err_parse fired must be destroyed explicitly. */ + RD_IF_FREE(inflight_acks, rd_list_destroy); + if (filtered_msgs) { + rd_kafka_op_t *rko; + int li; + RD_LIST_FOREACH(rko, filtered_msgs, li) + rd_kafka_op_destroy(rko); + rd_list_destroy(filtered_msgs); + } + + RD_IF_FREE(response_rko, rd_kafka_op_destroy); + *response_rko_out = NULL; + + if (rkt) + rd_kafka_topic_destroy0(rkt); + rd_rkb_dbg(rkb, MSG, "BADMSG", "Bad message (ShareFetch v%d): %s", + (int)request->rkbuf_reqhdr.ApiVersion, + rd_kafka_err2str(err)); + return err; +} + + +/** + * @brief Reset the share fetch session for a broker. + * + * Called when the broker returns SHARE_SESSION_NOT_FOUND, + * INVALID_SHARE_SESSION_EPOCH, or SHARE_SESSION_LIMIT_REACHED, + * indicating the session is lost or cannot be created. + * Resets the epoch to 0 and moves all toppars_in_session back + * to toppars_to_add so the next request re-establishes the + * full session. + * + * @param rkb Broker whose session is being reset. + * + * @locality broker thread + */ +static void rd_kafka_broker_session_reset(rd_kafka_broker_t *rkb) { + rd_kafka_toppar_t *rktp; + int i; + + rd_rkb_dbg( + rkb, FETCH, "SHARESESSION", + "Resetting share session: epoch %" PRId32 + " -> 0, " + "moving %d toppars_in_session to toppars_to_add", + rkb->rkb_share_fetch_session.epoch, + rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session)); + + rkb->rkb_share_fetch_session.epoch = 0; + + /* Remove toppars_to_forget from toppars_in_session — these + * were pending removal and should not be re-added to the + * new session. */ + if (rkb->rkb_share_fetch_session.toppars_to_forget) { + rd_kafka_toppar_t *forget_rktp; + + RD_LIST_FOREACH(forget_rktp, + rkb->rkb_share_fetch_session.toppars_to_forget, + i) { + rktp = rd_list_remove( + rkb->rkb_share_fetch_session.toppars_in_session, + forget_rktp); + if (rktp) + rd_kafka_toppar_destroy(rktp); + } + + rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_forget); + rkb->rkb_share_fetch_session.toppars_to_forget = NULL; + } + + /* Move remaining toppars_in_session to toppars_to_add so they + * get sent as new additions in the next request (epoch 0). */ + if (!rkb->rkb_share_fetch_session.toppars_to_add) { + rkb->rkb_share_fetch_session.toppars_to_add = + rkb->rkb_share_fetch_session.toppars_in_session; + rkb->rkb_share_fetch_session.toppars_in_session = + rd_list_new(0, rd_kafka_toppar_destroy_free); + } else { + while ((rktp = rd_list_pop( + rkb->rkb_share_fetch_session.toppars_in_session))) { + if (!rd_list_find( + rkb->rkb_share_fetch_session.toppars_to_add, + rktp, rd_list_cmp_ptr)) + rd_list_add( + rkb->rkb_share_fetch_session.toppars_to_add, + rktp); + else + rd_kafka_toppar_destroy(rktp); + } + } +} + +static void rd_kafka_broker_session_update_epoch(rd_kafka_broker_t *rkb) { + int32_t prev_epoch = rkb->rkb_share_fetch_session.epoch; + if (prev_epoch == -1) { + rd_rkb_dbg( + rkb, FETCH, "SHARESESSION", + "Not updating next epoch for -1 as it should be -1 again."); + return; + } + if (prev_epoch == INT32_MAX) + rkb->rkb_share_fetch_session.epoch = 1; + else + rkb->rkb_share_fetch_session.epoch++; + rd_rkb_dbg(rkb, FETCH, "SHARESESSION", + "share-fetch session epoch %" PRId32 " -> %" PRId32 + " (wrap=%s)", + prev_epoch, rkb->rkb_share_fetch_session.epoch, + prev_epoch == INT32_MAX ? "yes" : "no"); +} + +static void rd_kafka_broker_session_add_partition_to_toppars_in_session( + rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp) { + if (rd_list_find(rkb->rkb_share_fetch_session.toppars_in_session, rktp, + rd_list_cmp_ptr)) { + rd_rkb_dbg(rkb, FETCH, "SHARESESSION", + "%s [%" PRId32 "]: already in ShareFetch session", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition); + return; + } + rd_rkb_dbg(rkb, FETCH, "SHARESESSION", + "%s [%" PRId32 "]: adding to ShareFetch session", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition); + rd_kafka_toppar_keep(rktp); + rd_list_add(rkb->rkb_share_fetch_session.toppars_in_session, rktp); +} + +void rd_kafka_broker_session_remove_partition_from_toppars_in_session( + rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp) { + rd_kafka_toppar_t *removed_rktp; + removed_rktp = rd_list_remove( + rkb->rkb_share_fetch_session.toppars_in_session, rktp); + if (removed_rktp) { + rd_kafka_toppar_destroy(removed_rktp); + rd_rkb_dbg(rkb, FETCH, "SHARESESSION", + "%s [%" PRId32 "]: removed from ShareFetch session", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition); + } else { + rd_rkb_dbg(rkb, FETCH, "SHARESESSION", + "%s [%" PRId32 "]: not found in ShareFetch session", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition); + } +} + +static void +rd_kafka_broker_session_update_toppars_in_session(rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp, + rd_bool_t add) { + if (add) + rd_kafka_broker_session_add_partition_to_toppars_in_session( + rkb, rktp); + else + rd_kafka_broker_session_remove_partition_from_toppars_in_session( + rkb, rktp); +} + +static void rd_kafka_broker_session_update_toppars_list( + rd_kafka_broker_t *rkb, + rd_list_t **request_toppars_ptr, + rd_list_t **toppars_to_remove_from_ptr, + rd_bool_t add) { + size_t i; + rd_kafka_toppar_t *rktp, *removed_rktp; + rd_list_t *request_toppars = *request_toppars_ptr; + rd_list_t *toppars_to_remove_from = *toppars_to_remove_from_ptr; + + if (request_toppars == NULL || rd_list_cnt(request_toppars) == 0) + return; + + rd_rkb_dbg( + rkb, FETCH, "SHARESESSION", + "%d toppars being %s the session:", rd_list_cnt(request_toppars), + add ? "added to" : "removed from"); + + RD_LIST_FOREACH(rktp, request_toppars, i) { + rd_kafka_broker_session_update_toppars_in_session(rkb, rktp, + add); + if (toppars_to_remove_from) { + removed_rktp = + rd_list_remove(toppars_to_remove_from, rktp); + if (removed_rktp) { + rd_kafka_toppar_destroy( + removed_rktp); /* from partitions list */ + if (rd_list_empty(toppars_to_remove_from)) { + rd_list_destroy(toppars_to_remove_from); + toppars_to_remove_from = NULL; + *toppars_to_remove_from_ptr = NULL; + } + } + } + } + rd_list_destroy(request_toppars); + *request_toppars_ptr = NULL; +} + +static void +rd_kafka_broker_session_update_added_partitions(rd_kafka_broker_t *rkb) { + rd_kafka_broker_session_update_toppars_list( + rkb, &rkb->rkb_share_fetch_session.adding_toppars, + &rkb->rkb_share_fetch_session.toppars_to_add, rd_true); +} + +static void +rd_kafka_broker_session_update_removed_partitions(rd_kafka_broker_t *rkb) { + rd_kafka_broker_session_update_toppars_list( + rkb, &rkb->rkb_share_fetch_session.forgetting_toppars, + &rkb->rkb_share_fetch_session.toppars_to_forget, rd_false); +} + +static void rd_kafka_broker_session_update_partitions(rd_kafka_broker_t *rkb) { + rd_rkb_dbg( + rkb, FETCH, "SHARESESSION", + "applying session updates: adding_toppars=%d " + "forgetting_toppars=%d toppars_in_session=%d epoch=%" PRId32, + rkb->rkb_share_fetch_session.adding_toppars + ? rd_list_cnt(rkb->rkb_share_fetch_session.adding_toppars) + : 0, + rkb->rkb_share_fetch_session.forgetting_toppars + ? rd_list_cnt(rkb->rkb_share_fetch_session.forgetting_toppars) + : 0, + rkb->rkb_share_fetch_session.toppars_in_session + ? rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session) + : 0, + rkb->rkb_share_fetch_session.epoch); + rd_kafka_broker_session_update_added_partitions(rkb); + rd_kafka_broker_session_update_removed_partitions(rkb); +} + + +/** + * Update ShareFetch session state after a Fetch or ShareFetch response. + * TODO KIP-932: Improve efficiency of this function. + */ +static void rd_kafka_broker_session_update(rd_kafka_broker_t *rkb) { + rd_kafka_broker_session_update_epoch(rkb); + rd_kafka_broker_session_update_partitions(rkb); +} + +/** + * @brief Whether \p err is a per-partition ShareAcknowledge error + * whose CurrentLeader hint should trigger an inline metadata + * update. + */ +static rd_bool_t +rd_kafka_share_ack_err_is_leader_change(rd_kafka_resp_err_t err) { + switch (err) { + case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER: + case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH: + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART: + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: + return rd_true; + default: + return rd_false; + } +} + +/** + * @brief Parse a ShareAcknowledge response. + * + * ShareAcknowledge response contains per-partition error codes for + * acknowledgement results. Returns the top-level error code and sets + * the err field on each matching batch in \p ack_details. + * + * TODO KIP-932: Consider using rd_kafka_error_t instead of + * rd_kafka_resp_err_t for the per-partition result and + * acknowledgement callback. + * + * @param rkb Broker handle. + * @param rkbuf Response buffer. + * @param request Original request buffer. + * @param ack_details List of acknowledgement batches sent in the + * request. Each matched batch's rktpar->err is + * updated with the partition-level error code + * from the response. + * + * @returns Top-level error code. + * @locality broker thread + */ +static rd_kafka_resp_err_t +rd_kafka_share_acknowledge_reply_handle(rd_kafka_broker_t *rkb, + rd_kafka_buf_t *rkbuf, + rd_kafka_buf_t *request, + rd_list_t *ack_details) { + int32_t TopicArrayCnt; + int i; + const int log_decode_errors = LOG_ERR; + int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_kafkap_str_t ErrorStr = RD_KAFKAP_STR_INITIALIZER_EMPTY; + rd_kafkap_NodeEndpoints_t NodeEndpoints; + rd_list_t *leader_changes = NULL; + NodeEndpoints.NodeEndpoints = NULL; + NodeEndpoints.NodeEndpointCnt = 0; + + rd_kafka_buf_read_throttle_time(rkbuf); + + rd_kafka_buf_read_i16(rkbuf, &ErrorCode); + rd_kafka_buf_read_str(rkbuf, &ErrorStr); + + if (ErrorCode) { + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "ShareAcknowledge response error %s: '%.*s'", + rd_kafka_err2name(ErrorCode), + RD_KAFKAP_STR_PR(&ErrorStr)); + return ErrorCode; + } + + rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX); + + for (i = 0; i < TopicArrayCnt; i++) { + rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO; + int32_t PartitionArrayCnt; + int j; + + rd_kafka_buf_read_uuid(rkbuf, &topic_id); + + rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt, + RD_KAFKAP_PARTITIONS_MAX); + + for (j = 0; j < PartitionArrayCnt; j++) { + int32_t Partition; + int16_t PartErrorCode; + rd_kafkap_str_t PartErrorStr; + rd_kafkap_CurrentLeader_t CurrentLeader; + rd_kafka_share_ack_batches_t *batch; + rd_kafkap_share_leader_change_t *lc; + + rd_kafka_buf_read_i32(rkbuf, &Partition); + rd_kafka_buf_read_i16(rkbuf, &PartErrorCode); + rd_kafka_buf_read_str(rkbuf, &PartErrorStr); + + /* CurrentLeader */ + rd_kafka_buf_read_CurrentLeader(rkbuf, &CurrentLeader); + + batch = rd_kafka_share_find_ack_batch_by_id( + ack_details, topic_id, Partition); + if (batch) { + /* Conditional on _IN_PROGRESS as a + * defensive safety net — the broker's + * per-partition PartErrorCode is the + * authoritative result for partitions + * actually sent in the request. */ + if (batch->rktpar->err == + RD_KAFKA_RESP_ERR__IN_PROGRESS) + batch->rktpar->err = PartErrorCode; + } else { + rd_rkb_log( + rkb, LOG_ERR, "SHAREACK", + "Invalid partition %" PRId32 + " received in ShareAcknowledge response", + Partition); + } + + if (PartErrorCode) { + /* TODO KIP-932: write test cases for each + * per-partition ShareAcknowledge error code + * once the mock cluster exposes a + * per-partition error-injection API for + * ShareAcknowledge responses. Today only + * NOT_LEADER_OR_FOLLOWER (via + * rd_kafka_mock_partition_set_leader) is + * deterministically reachable; the other + * leader-change errors + * (FENCED_LEADER_EPOCH, + * UNKNOWN_TOPIC_OR_PART, + * UNKNOWN_TOPIC_ID), + * INVALID_RECORD_STATE, + * INVALID_REQUEST, KAFKA_STORAGE_ERROR + * and the inline-leader-update path have + * no deterministic mock trigger. */ + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "ShareAcknowledge partition %" PRId32 + " error %s: '%.*s'", + Partition, + rd_kafka_err2name(PartErrorCode), + RD_KAFKAP_STR_PR(&PartErrorStr)); + + if (rd_kafka_share_ack_err_is_leader_change( + PartErrorCode) && + CurrentLeader.LeaderId != -1 && + CurrentLeader.LeaderEpoch != -1) { + if (!leader_changes) + leader_changes = + rd_list_new(0, rd_free); + lc = rd_calloc(1, sizeof(*lc)); + lc->topic_id = topic_id; + lc->partition = Partition; + lc->current_leader = CurrentLeader; + rd_list_add(leader_changes, lc); + } + } + + /* Partition tags */ + rd_kafka_buf_skip_tags(rkbuf); + } + + /* Topic tags */ + rd_kafka_buf_skip_tags(rkbuf); + } + + rd_kafka_buf_read_NodeEndpoints(rkbuf, &NodeEndpoints); + + /* Top level tags */ + rd_kafka_buf_skip_tags(rkbuf); + + rd_kafkap_share_leader_changes_apply(rkb, leader_changes, + &NodeEndpoints); + + RD_IF_FREE(leader_changes, rd_list_destroy); + RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free); + return RD_KAFKA_RESP_ERR_NO_ERROR; + +err_parse: + RD_IF_FREE(leader_changes, rd_list_destroy); + RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free); + rd_rkb_dbg(rkb, MSG, "BADMSG", + "Bad ShareAcknowledge response (v%d): parse error", + (int)request->rkbuf_reqhdr.ApiVersion); + return rkbuf->rkbuf_err; +} + + +/** + * @brief ShareAcknowledge response handling callback. + * + * @locality broker thread (or any thread if err == __DESTROY). + */ +static void rd_kafka_broker_share_acknowledge_reply(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *reply, + rd_kafka_buf_t *request, + void *opaque) { + rd_kafka_op_t *rko_orig = opaque; + + /* Parse and handle the response (unless the request errored). + * The parser writes per-partition err on matching batches in + * ack_details (only on batches still at the _IN_PROGRESS + * sentinel as a defensive safety net). DESTROY case falls + * through here and is handled by the generic top-level error + * path below. + * + * If the parser ran and succeeded (err remains 0), convert + * any batch still at _IN_PROGRESS to INVALID_RECORD_STATE — + * those partitions were sent in the request but missing from + * the response. + * + * If the parser was skipped or failed (err != 0), leave + * batches at _IN_PROGRESS so the helper at the end can + * propagate the top-level err to them. */ + if (!err && reply) { + err = rd_kafka_share_acknowledge_reply_handle( + rkb, reply, request, + rko_orig->rko_u.share_fetch.ack_details); + if (!err && rko_orig->rko_u.share_fetch.ack_details) { + rd_kafka_share_ack_batches_t *batch; + int i; + RD_LIST_FOREACH( + batch, rko_orig->rko_u.share_fetch.ack_details, i) { + if (batch->rktpar->err == + RD_KAFKA_RESP_ERR__IN_PROGRESS) + batch->rktpar->err = + RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE; + } + } + } + + rd_kafka_broker_session_update(rkb); + + if (unlikely(err)) { + rd_rkb_log(rkb, LOG_INFO, "SHAREACK", + "ShareAcknowledge reply error: %s", + rd_kafka_err2str(err)); + switch (err) { + case RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND: + case RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH: + case RD_KAFKA_RESP_ERR__TRANSPORT: + case RD_KAFKA_RESP_ERR__TIMED_OUT: + /* __TRANSPORT means connection is already dead. + * On __TIMED_OUT the connection is torn down by the + * request-timeout scan (socket.max.fails is forced to + * 1 for share consumers and cannot be changed) and the + * broker reconnects via the share-serve persistent + * connection driver; here we only reset the session so + * it re-establishes at epoch 0. */ + rd_kafka_broker_session_reset(rkb); + break; + + case RD_KAFKA_RESP_ERR__BAD_MSG: + case RD_KAFKA_RESP_ERR__UNDERFLOW: + /* Wire-level parse failure: response envelope or + * an inner MessageSet/record was malformed or + * truncated. Indicates broker bug, on-the-wire + * corruption, or version mismatch — log loudly so + * the user notices, since the LOG_INFO top-level + * log can be easy to miss in production output. */ + rd_rkb_log( + rkb, LOG_ERR, "SHAREACK", + "ShareAcknowledge response parse failure: %s " + "(ApiVersion %hd) — broker bug, wire " + "corruption, or version mismatch; " + "this response is being dropped", + rd_kafka_err2str(err), + request->rkbuf_reqhdr.ApiVersion); + break; + + default: + /* No retry for ShareAcknowledge RPC-level + * errors. The error semantics (partition + * moved, session state lost, etc.) make a + * blind retry unlikely to succeed and risk + * duplicating side effects. */ + break; + } + } + + if (rko_orig->rko_u.share_fetch.should_leave) + rd_kafka_broker_share_fetch_session_clear(rkb); + + /* ack_details is owned by the op and freed by the op destructor + * after the main thread has processed the reply. */ + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(rko_orig, + err); +} + + +/** + * @broker ShareFetchResponse handling. + * + * @locality broker thread (or any thread if err == __DESTROY). + */ +static void rd_kafka_broker_share_fetch_reply(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *reply, + rd_kafka_buf_t *request, + void *opaque) { + + rd_kafka_op_t *rko_orig = opaque; + rd_kafka_op_t *response_rko = NULL; + + /* Parse the response only if the network/broker layer didn't + * report an error. If err is set (e.g. __TRANSPORT, + * __TIMED_OUT, __DESTROY), the reply buffer is unusable so + * we skip parsing. The parser writes per-partition err on + * matching batches in ack_details (only on batches still at + * the _IN_PROGRESS sentinel, so any deliberately pre-set + * value such as INVALID_SHARE_SESSION_EPOCH is preserved). + * The DESTROY/network error case falls through to the + * generic top-level error path below. + * + * If the parser ran and succeeded (err remains 0), convert + * any batch still at _IN_PROGRESS to INVALID_RECORD_STATE — + * those partitions were sent in the request but missing from + * the response. + * + * If the parser was skipped or failed (err != 0), leave + * batches at _IN_PROGRESS so the helper at the end can + * propagate the top-level err to them. */ + if (!err && reply) { + err = rd_kafka_share_fetch_reply_handle(rkb, reply, request, + &response_rko); + if (!err && rko_orig->rko_u.share_fetch.ack_details) { + rd_kafka_share_ack_batches_t *batch; + int i; + RD_LIST_FOREACH( + batch, rko_orig->rko_u.share_fetch.ack_details, i) { + if (batch->rktpar->err == + RD_KAFKA_RESP_ERR__IN_PROGRESS) + batch->rktpar->err = + RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE; + } + } + } + + rd_kafka_broker_session_update(rkb); + + if (unlikely(err)) { + rd_rkb_log(rkb, LOG_INFO, "SHAREFETCH", + "ShareFetch reply error: %s", rd_kafka_err2str(err)); + switch (err) { + case RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND: + case RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH: + case RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED: + case RD_KAFKA_RESP_ERR__TRANSPORT: + case RD_KAFKA_RESP_ERR__TIMED_OUT: + /* Session is invalid, lost, cannot be created, + * or connection/request failed. + * Reset session state so the next request + * re-establishes a new session (epoch 0). + * __TRANSPORT means connection is already dead. + * On __TIMED_OUT the connection is torn down by the + * request-timeout scan (socket.max.fails is forced to + * 1 for share consumers and cannot be changed) and the + * broker reconnects via the share-serve persistent + * connection driver. */ + rd_kafka_broker_session_reset(rkb); + break; + + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: { + char tmp[128]; + rd_snprintf(tmp, sizeof(tmp), "ShareFetch failed: %s", + rd_kafka_err2str(err)); + rd_kafka_metadata_refresh_known_topics( + rkb->rkb_rk, NULL, rd_false /*!force*/, tmp); + break; + } + + case RD_KAFKA_RESP_ERR__BAD_MSG: + case RD_KAFKA_RESP_ERR__UNDERFLOW: + /* Wire-level parse failure: response envelope or + * an inner MessageSet/record was malformed or + * truncated. Indicates broker bug, on-the-wire + * corruption, or version mismatch — log loudly so + * the user notices, since the LOG_INFO top-level + * log can be easy to miss in production output. */ + rd_rkb_log(rkb, LOG_ERR, "SHAREFETCH", + "ShareFetch response parse failure: %s " + "(ApiVersion %hd) — broker bug, wire " + "corruption, or version mismatch; " + "this response is being dropped", + rd_kafka_err2str(err), + request->rkbuf_reqhdr.ApiVersion); + break; + + default: + /* No error-specific handling at the request + * level. Non-session top-level errors are + * treated as transient — the main thread + * retries by selecting another broker. */ + break; + } + + /* There is no retry for ShareFetch RPC at the broker + * thread level. */ + } + + /* ack_details is owned by the op and freed by the op destructor + * after the main thread has processed the reply. */ + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(rko_orig, + err); + + /* Enqueue the response for the app thread AFTER sending the + * reply to the main thread. This ensures the main thread + * processes the reply (resetting share_fetch_more_records) + * before the app thread wakes up and enqueues a new FANOUT. */ + if (response_rko) + rd_kafka_q_enq(rkb->rkb_rk->rk_cgrp->rkcg_q, response_rko); +} + +/** + * @broker FetchResponse handling. + * + * @locality broker thread (or any thread if err == __DESTROY). + */ +static void rd_kafka_broker_fetch_reply(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *reply, + rd_kafka_buf_t *request, + void *opaque) { + + if (err == RD_KAFKA_RESP_ERR__DESTROY) + return; /* Terminating */ + + rd_kafka_assert(rkb->rkb_rk, rkb->rkb_fetching > 0); + rkb->rkb_fetching = 0; + + /* Parse and handle the messages (unless the request errored) */ + if (!err && reply) + err = rd_kafka_fetch_reply_handle(rkb, reply, request); + + if (unlikely(err)) { + char tmp[128]; + + rd_rkb_dbg(rkb, MSG, "FETCH", "Fetch reply: %s", + rd_kafka_err2str(err)); + switch (err) { + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART: + case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION: + case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE: + case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: + /* Request metadata information update */ + rd_snprintf(tmp, sizeof(tmp), "FetchRequest failed: %s", + rd_kafka_err2str(err)); + rd_kafka_metadata_refresh_known_topics( + rkb->rkb_rk, NULL, rd_true /*force*/, tmp); + /* FALLTHRU */ + + case RD_KAFKA_RESP_ERR__TRANSPORT: + case RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT: + case RD_KAFKA_RESP_ERR__MSG_TIMED_OUT: + /* The fetch is already intervalled from + * consumer_serve() so dont retry. */ + break; + + default: + break; + } + + rd_kafka_broker_fetch_backoff(rkb, err); + /* FALLTHRU */ + } +} + +/** + * @brief Check if any toppars have a zero topic id. + * + */ +static rd_bool_t can_use_topic_ids(rd_kafka_broker_t *rkb) { + rd_kafka_toppar_t *rktp = rkb->rkb_active_toppar_next; + do { + if (RD_KAFKA_UUID_IS_ZERO(rktp->rktp_rkt->rkt_topic_id)) + return rd_false; + } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp, + rktp_activelink)) != + rkb->rkb_active_toppar_next); + + return rd_true; +} + + +/** + * @brief Sum the total acknowledgement entries and records across all + * per-partition batches in \p ack_details. + * + * @param ack_details List of rd_kafka_share_ack_batches_t, or NULL. + * @param entries_out Set to the total number of ack entries (offset ranges). + * @param records_out Set to the total number of acknowledged records + * (sum of entry->size across all entries). + */ +static void rd_kafka_share_ack_details_totals(rd_list_t *ack_details, + int *entries_out, + int64_t *records_out) { + rd_kafka_share_ack_batches_t *batches; + rd_kafka_share_ack_batch_entry_t *entry; + int k, j; + int entries = 0; + int64_t records = 0; + + if (ack_details) { + RD_LIST_FOREACH(batches, ack_details, k) { + entries += rd_list_cnt(&batches->entries); + RD_LIST_FOREACH(entry, &batches->entries, j) { + records += entry->size; + } + } + } + + *entries_out = entries; + *records_out = records; +} + + +void rd_kafka_ShareFetchRequest(rd_kafka_broker_t *rkb, + const rd_kafkap_str_t *group_id, + const rd_kafkap_str_t *member_id, + int32_t share_session_epoch, + int32_t wait_max_ms, + int32_t min_bytes, + int32_t max_bytes, + int32_t max_records, + int32_t batch_size, + rd_list_t *toppars_to_add, + rd_list_t *toppars_to_forget, + rd_kafka_op_t *rko_orig, + rd_ts_t now) { + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + int cnt = 0; + size_t of_TopicArrayCnt = 0; + int TopicArrayCnt = 0; + size_t of_PartitionArrayCnt = 0; + int PartitionArrayCnt = 0; + rd_kafka_topic_t *rkt_last = NULL; + int16_t ApiVersion = 0; + size_t rkbuf_size = 0; + int toppars_to_add_cnt = + toppars_to_add ? rd_list_cnt(toppars_to_add) : 0; + int i; + rd_list_t *ack_details = + rko_orig ? rko_orig->rko_u.share_fetch.ack_details : NULL; + int ack_details_cnt = ack_details ? rd_list_cnt(ack_details) : 0; + /* TODO KIP-932: Ensure there is no intersection between toppars_to_add + * and ack_details. A toppar should not appear in both lists. */ + int total_ack_entries = 0; + int64_t total_ack_records = 0; + int toppars_to_forget_cnt = + toppars_to_forget ? rd_list_cnt(toppars_to_forget) : 0; + rd_bool_t is_fetching_messages = max_records > 0 ? rd_true : rd_false; + /* FirstOffset + LastOffset + AcknowledgementType per ack entry */ + size_t acknowledgement_size = 8 + 8 + 1; + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "toppars_to_add_cnt=%d, ack_details_cnt=%d, " + "total_ack_entries=%d, toppars_to_forget_cnt=%d, " + "is_fetching_messages=%d", + toppars_to_add_cnt, ack_details_cnt, total_ack_entries, + toppars_to_forget_cnt, is_fetching_messages); + + /* Calculate buffer size */ + if (group_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(group_id); + if (member_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(member_id); + /* ShareSessionEpoch + WaitMaxMs + MinBytes + MaxBytes + MaxRecords + + * BatchSize + TopicArrayCnt*/ + rkbuf_size += 4 + 4 + 4 + 4 + 4 + 4 + 4; + /* N x (topic id + partition id) for topics to add */ + rkbuf_size += (toppars_to_add_cnt * (32 + 4)); + /* M x (topic id + partition id) for ack details partitions + + * E x acknowledgement entries */ + + rkbuf_size += (ack_details_cnt * (32 + 4)); + /* Sum total ack entries and total records across all + * ack_details batches. */ + rd_kafka_share_ack_details_totals(ack_details, &total_ack_entries, + &total_ack_records); + /* Accumulate piggybacked record-level acknowledgements for + * consumer.share.fetch.manager.acknowledgements.send.{total,rate}. */ + if (total_ack_records > 0) + rd_atomic64_add( + &rkb->rkb_rk->rk_telemetry.acknowledgements_send_total, + total_ack_records); + + rkbuf_size += (total_ack_entries * acknowledgement_size); + + /* F x (topic id + partition id) for topics to forget */ + rkbuf_size += (toppars_to_forget_cnt * (32 + 4)); + + ApiVersion = rd_kafka_broker_ApiVersion_supported( + rkb, RD_KAFKAP_ShareFetch, 1, 1, NULL); + + rkbuf = rd_kafka_buf_new_flexver_request(rkb, RD_KAFKAP_ShareFetch, 1, + rkbuf_size, rd_true); + + if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2) + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, + RD_KAFKA_FEATURE_MSGVER2); + else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1) + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, + RD_KAFKA_FEATURE_MSGVER1); + else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME) + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, + RD_KAFKA_FEATURE_THROTTLETIME); + + /* GroupId */ + rd_kafka_buf_write_kstr(rkbuf, group_id); + + /* MemberId */ + rd_kafka_buf_write_kstr(rkbuf, member_id); + + // printf(" --------------------------------------- + // rd_kafka_ShareFetchRequest: member_id=%.*s\n", + // RD_KAFKAP_STR_PR(member_id)); + + /* ShareSessionEpoch */ + rd_kafka_buf_write_i32(rkbuf, share_session_epoch); + + /* WaitMaxMs */ + rd_kafka_buf_write_i32(rkbuf, wait_max_ms); + + /* MinBytes */ + rd_kafka_buf_write_i32(rkbuf, min_bytes); + + /* MaxBytes */ + rd_kafka_buf_write_i32(rkbuf, max_bytes); + + /* MaxRecords */ + rd_kafka_buf_write_i32(rkbuf, max_records); + + /* BatchSize */ + rd_kafka_buf_write_i32(rkbuf, batch_size); + + /* Write zero TopicArrayCnt but store pointer for later update */ + of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf); + + /* TODO KIP-932: Ensure toppars_to_add and ack_details don't have + * common rktps. A toppar should only appear in one list. + * Also merge toppars_to_add and ack_details as both can have + * the same topic but different partitions. */ + + /* Write toppars_to_add: new toppars being added to session + * (no acknowledgements for newly added toppars) */ + if (toppars_to_add) { + /* TODO KIP-932: This condition will cause partitions of same + topics to be inside single instance of the topic as + toppars_to_add is not sorted. Eg: T1 0, T1 1, T2 0, T1 3, T1 + 5, T2 1 will translate to T1 (0,1), T2 (0), T1 (3, 5), T2 + (1) instead it should be T1 (0,1,3,5) T2(0,1) Fix this. */ + RD_LIST_FOREACH(rktp, toppars_to_add, i) { + if (rkt_last != rktp->rktp_rkt) { + if (rkt_last != NULL) { + /* Update PartitionArrayCnt */ + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, + PartitionArrayCnt); + /* Topic tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + rd_kafka_topic_rdlock(rktp->rktp_rkt); + /* Topic ID */ + rd_kafka_buf_write_uuid( + rkbuf, &rktp->rktp_rkt->rkt_topic_id); + rd_kafka_topic_rdunlock(rktp->rktp_rkt); + + TopicArrayCnt++; + rkt_last = rktp->rktp_rkt; + /* Partition count */ + of_PartitionArrayCnt = + rd_kafka_buf_write_arraycnt_pos(rkbuf); + PartitionArrayCnt = 0; + } + + PartitionArrayCnt++; + + /* Partition */ + rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition); + + /* No acknowledgements for newly added toppars */ + rd_kafka_buf_write_arraycnt(rkbuf, 0); + + /* Partition tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Share Fetch adding topic %.*s [%" PRId32 + "]", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition); + + cnt++; + } + + /* Finalize last topic from toppars_to_add */ + if (rkt_last != NULL) { + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, PartitionArrayCnt); + rd_kafka_buf_write_tags_empty(rkbuf); + } + } + + /* Write ack_details: toppars with acknowledgement batches. + * Uses rktpar directly (no rktp/lock needed). */ + if (ack_details) { + rd_kafka_Uuid_t *topic_id_last = NULL; + rd_kafka_share_ack_batches_t *batches; + rd_kafka_topic_partition_private_t *parpriv; + rd_kafka_share_ack_batch_entry_t *entry; + int k, m, entries_cnt; + + /* TODO KIP-932: This condition will cause partitions of same + topics to be inside single instance of the topic as + ack_details is not sorted. Eg: T1 0, T1 1, T2 0, T1 3, T1 + 5, T2 1 will translate to T1 (0,1), T2 (0), T1 (3, 5), T2 + (1) instead it should be T1 (0,1,3,5) T2(0,1) Fix this. */ + PartitionArrayCnt = 0; + + RD_LIST_FOREACH(batches, ack_details, k) { + /* TODO KIP-932: Ensure rktpar and rktp are in sync, + * leader has not changed in between, and rktp is + * present before reaching this point. + * Also ensure batches->entries are not empty. */ + /* _private should always be present here as it was + * set when building the ack batches from the + * inflight map. */ + parpriv = (rd_kafka_topic_partition_private_t *) + batches->rktpar->_private; + rd_dassert(parpriv != NULL); + + if (topic_id_last == NULL || + rd_kafka_Uuid_cmp(*topic_id_last, + parpriv->topic_id) != 0) { + if (topic_id_last != NULL) { + /* Update PartitionArrayCnt */ + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, + PartitionArrayCnt); + /* Topic tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + /* Topic ID */ + rd_kafka_buf_write_uuid(rkbuf, + &parpriv->topic_id); + + TopicArrayCnt++; + topic_id_last = &parpriv->topic_id; + /* Partition count */ + of_PartitionArrayCnt = + rd_kafka_buf_write_arraycnt_pos(rkbuf); + PartitionArrayCnt = 0; + } + + PartitionArrayCnt++; + + /* Partition */ + rd_kafka_buf_write_i32(rkbuf, + batches->rktpar->partition); + + /* Write acknowledgement batches */ + entries_cnt = rd_list_cnt(&batches->entries); + rd_kafka_buf_write_arraycnt(rkbuf, entries_cnt); + + RD_LIST_FOREACH(entry, &batches->entries, m) { + /* FirstOffset */ + rd_kafka_buf_write_i64(rkbuf, + entry->start_offset); + /* LastOffset */ + rd_kafka_buf_write_i64(rkbuf, + entry->end_offset); + /* AcknowledgeTypes */ + rd_kafka_buf_write_arraycnt(rkbuf, 1); + rd_kafka_buf_write_i8(rkbuf, + (int8_t)entry->types[0]); + /* Acknowledgement tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + /* Partition tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Share Fetch ack for topic %s [%" PRId32 + "] with %d entries", + batches->rktpar->topic, + batches->rktpar->partition, entries_cnt); + + cnt++; + } + + /* Finalize last topic from ack_details */ + if (topic_id_last != NULL) { + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, PartitionArrayCnt); + rd_kafka_buf_write_tags_empty(rkbuf); + } + } + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Share Fetch Request (epoch %" PRId32 + ") with %d toppars on %d topics", + rkb->rkb_share_fetch_session.epoch, cnt, TopicArrayCnt); + + /* Update TopicArrayCnt */ + rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt); + + if (toppars_to_forget) { + TopicArrayCnt = 0; + PartitionArrayCnt = 0; + rkt_last = NULL; + /* Write zero TopicArrayCnt but store pointer for later update + */ + of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf); + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", "Forgetting %d toppars", + toppars_to_forget_cnt); + RD_LIST_FOREACH(rktp, toppars_to_forget, i) { + /* TODO KIP-932: This condition will cause partitions of + same topics to be inside single instance of the topic as + toppars_to_forget is not sorted. Eg: T1 0, T1 1, T2 0, + T1 3, T1 5, T2 1 will translate to T1 (0,1), T2 (0), + T1 (3, 5), T2 (1) instead it should be T1 (0,1,3,5) + T2(0,1) Fix this. */ + if (rkt_last != rktp->rktp_rkt) { + if (rkt_last != NULL) { + /* Update PartitionArrayCnt */ + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, + PartitionArrayCnt); + /* Topic tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + rd_kafka_topic_rdlock(rktp->rktp_rkt); + /* Topic ID */ + rd_kafka_buf_write_uuid( + rkbuf, &rktp->rktp_rkt->rkt_topic_id); + rd_kafka_topic_rdunlock(rktp->rktp_rkt); + + TopicArrayCnt++; + rkt_last = rktp->rktp_rkt; + /* Partition count */ + of_PartitionArrayCnt = + rd_kafka_buf_write_arraycnt_pos(rkbuf); + PartitionArrayCnt = 0; + } + + PartitionArrayCnt++; + + /* Partition */ + rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition); + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Forgetting Fetch partition %.*s [%" PRId32 + "]", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition); + } + if (rkt_last != NULL) { + /* Update last topic's PartitionArrayCnt */ + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, PartitionArrayCnt); + /* Topic tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + /* Update TopicArrayCnt */ + rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, + TopicArrayCnt); + } else { + /* ForgottenToppars */ + rd_kafka_buf_write_arraycnt(rkbuf, 0); + } + + /* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */ + if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000) + rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING; + + /* Use configured timeout */ + rd_kafka_buf_set_timeout(rkbuf, + rkb->rkb_rk->rk_conf.socket_timeout_ms + + rkb->rkb_rk->rk_conf.fetch_wait_max_ms, + now); + + /* Copy toppars_to_add/toppars_to_forget into adding/forgetting lists + * just before sending. On response, session_update() will move + * adding_toppars into toppars_in_session and remove from + * toppars_to_add. */ + if (rkb->rkb_share_fetch_session.toppars_to_add) + rkb->rkb_share_fetch_session.adding_toppars = + rd_list_copy(rkb->rkb_share_fetch_session.toppars_to_add, + rd_kafka_toppar_list_copy, NULL); + if (rkb->rkb_share_fetch_session.toppars_to_forget) + rkb->rkb_share_fetch_session.forgetting_toppars = + rd_list_copy(rkb->rkb_share_fetch_session.toppars_to_forget, + rd_kafka_toppar_list_copy, NULL); + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Issuing ShareFetch request (max wait %dms, min %d bytes, " + "max %d bytes, max %d records) with %d toppars", + wait_max_ms, min_bytes, max_bytes, max_records, cnt); + rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_share_fetch_reply, + rko_orig); + + return; +} + +/** + * @brief Build and send a ShareAcknowledge request. + * + * Used for ack-only requests (no fetching) and session close (epoch=-1). + * ShareAcknowledge carries GroupId, MemberId, ShareSessionEpoch, and + * acknowledgement batches only — no fetch parameters or forgotten topics. + * + * @param rkb Broker to send request to. + * @param group_id Consumer group id. + * @param member_id Consumer member id. + * @param share_session_epoch Session epoch (-1 for close). + * @param rko_orig The originating SHARE_FETCH op (carries ack_details). + * @param now Current timestamp. + * + * @locality broker thread + */ +void rd_kafka_ShareAcknowledgeRequest(rd_kafka_broker_t *rkb, + const rd_kafkap_str_t *group_id, + const rd_kafkap_str_t *member_id, + int32_t share_session_epoch, + rd_kafka_op_t *rko_orig, + rd_ts_t now) { + rd_kafka_buf_t *rkbuf; + int cnt = 0; + size_t of_TopicArrayCnt = 0; + int TopicArrayCnt = 0; + size_t of_PartitionArrayCnt = 0; + int PartitionArrayCnt = 0; + size_t rkbuf_size = 0; + rd_list_t *ack_details = + rko_orig ? rko_orig->rko_u.share_fetch.ack_details : NULL; + int ack_details_cnt = ack_details ? rd_list_cnt(ack_details) : 0; + int total_ack_entries = 0; + int64_t total_ack_records = 0; + /* FirstOffset + LastOffset + AcknowledgementType per ack entry */ + size_t acknowledgement_size = 8 + 8 + 1; + + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "ack_details_cnt=%d, share_session_epoch=%" PRId32, + ack_details_cnt, share_session_epoch); + + /* Calculate buffer size */ + if (group_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(group_id); + if (member_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(member_id); + /* ShareSessionEpoch + TopicArrayCnt */ + rkbuf_size += 4 + 4; + /* N x (topic id + partition id) for ack details */ + rkbuf_size += (ack_details_cnt * (32 + 4)); + /* Sum total ack entries and total records across all + * ack_details batches. */ + rd_kafka_share_ack_details_totals(ack_details, &total_ack_entries, + &total_ack_records); + /* Accumulate standalone record-level acknowledgements for + * consumer.share.fetch.manager.acknowledgements.send.{total,rate}. */ + if (total_ack_records > 0) + rd_atomic64_add( + &rkb->rkb_rk->rk_telemetry.acknowledgements_send_total, + total_ack_records); + + rkbuf_size += (total_ack_entries * acknowledgement_size); + + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_ShareAcknowledge, 1, rkbuf_size, rd_true); + + rd_kafka_buf_ApiVersion_set( + rkbuf, + rd_kafka_broker_ApiVersion_supported( + rkb, RD_KAFKAP_ShareAcknowledge, 1, 1, NULL), + 0); + + /* GroupId */ + rd_kafka_buf_write_kstr(rkbuf, group_id); + + /* MemberId */ + rd_kafka_buf_write_kstr(rkbuf, member_id); + + /* ShareSessionEpoch */ + rd_kafka_buf_write_i32(rkbuf, share_session_epoch); + + /* Topics array with acknowledgement batches */ + of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf); + + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "Building ShareAcknowledge request with %d ack toppars " + "and %d total ack entries", + ack_details_cnt, total_ack_entries); + + if (ack_details) { + rd_kafka_Uuid_t *topic_id_last = NULL; + rd_kafka_share_ack_batches_t *batches; + rd_kafka_topic_partition_private_t *parpriv; + rd_kafka_share_ack_batch_entry_t *entry; + int k, m, entries_cnt; + + RD_LIST_FOREACH(batches, ack_details, k) { + parpriv = (rd_kafka_topic_partition_private_t *) + batches->rktpar->_private; + rd_dassert(parpriv != NULL); + + if (topic_id_last == NULL || + rd_kafka_Uuid_cmp(*topic_id_last, + parpriv->topic_id) != 0) { + if (topic_id_last != NULL) { + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, + PartitionArrayCnt); + rd_kafka_buf_write_tags_empty(rkbuf); + } + + /* Topic ID */ + rd_kafka_buf_write_uuid(rkbuf, + &parpriv->topic_id); + + TopicArrayCnt++; + topic_id_last = &parpriv->topic_id; + of_PartitionArrayCnt = + rd_kafka_buf_write_arraycnt_pos(rkbuf); + PartitionArrayCnt = 0; + } + + PartitionArrayCnt++; + + /* Partition */ + rd_kafka_buf_write_i32(rkbuf, + batches->rktpar->partition); + + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "Adding ack for topic %s [%" PRId32 + "] with %d entries", + batches->rktpar->topic, + batches->rktpar->partition, + rd_list_cnt(&batches->entries)); + + /* Write acknowledgement batches */ + entries_cnt = rd_list_cnt(&batches->entries); + rd_kafka_buf_write_arraycnt(rkbuf, entries_cnt); + + RD_LIST_FOREACH(entry, &batches->entries, m) { + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "Adding ack entry with start offset " + "%" PRId64 ", end offset %" PRId64 + ", type %d", + entry->start_offset, + entry->end_offset, entry->types[0]); + /* FirstOffset */ + rd_kafka_buf_write_i64(rkbuf, + entry->start_offset); + /* LastOffset */ + rd_kafka_buf_write_i64(rkbuf, + entry->end_offset); + /* AcknowledgeTypes */ + rd_kafka_buf_write_arraycnt(rkbuf, 1); + rd_kafka_buf_write_i8(rkbuf, + (int8_t)entry->types[0]); + /* Acknowledgement tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + /* Partition tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + + cnt++; + } + + /* Finalize last topic from ack_details */ + if (topic_id_last != NULL) { + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, PartitionArrayCnt); + rd_kafka_buf_write_tags_empty(rkbuf); + } + } + + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "ShareAcknowledge Request with %d toppars on %d topics", cnt, + TopicArrayCnt); + + /* Update TopicArrayCnt */ + rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt); + + /* Wire RPC timeout is socket.timeout.ms, decoupled from any + * caller-side commit_sync deadline. Interactions covered by + * tests/0182-share_consumer_error_handling_mock.c + * (do_test_socket_timeout_full_ack_then_more). */ + rd_kafka_buf_set_timeout(rkbuf, rkb->rkb_rk->rk_conf.socket_timeout_ms, + now); + + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "Issuing ShareAcknowledge request with %d toppars", cnt); + rd_kafka_broker_buf_enq1( + rkb, rkbuf, rd_kafka_broker_share_acknowledge_reply, rko_orig); +} + + +void rd_kafka_broker_share_fetch_session_clear(rd_kafka_broker_t *rkb) { + /* Clear toppars in session */ + rd_rkb_dbg( + rkb, BROKER, "SHARESESSION", + "Clearing %d toppars from ShareFetch session", + rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session)); + rd_list_clear(rkb->rkb_share_fetch_session.toppars_in_session); + + /* Clear toppars to add */ + if (rkb->rkb_share_fetch_session.toppars_to_add) { + rd_rkb_dbg( + rkb, BROKER, "SHARESESSION", + "Clearing %d toppars to add from ShareFetch session on " + "clear", + rd_list_cnt(rkb->rkb_share_fetch_session.toppars_to_add)); + rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_add); + rkb->rkb_share_fetch_session.toppars_to_add = NULL; + } + + /* Clear toppars to forget */ + if (rkb->rkb_share_fetch_session.toppars_to_forget) { + rd_rkb_dbg(rkb, BROKER, "SHARESESSION", + "Clearing %d toppars to forget from ShareFetch " + "session on clear", + rd_list_cnt( + rkb->rkb_share_fetch_session.toppars_to_forget)); + rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_forget); + rkb->rkb_share_fetch_session.toppars_to_forget = NULL; + } + + /* Clear adding toppars */ + if (rkb->rkb_share_fetch_session.adding_toppars) { + rd_rkb_dbg( + rkb, BROKER, "SHARESESSION", + "Clearing %d adding toppars from ShareFetch session on " + "clear", + rd_list_cnt(rkb->rkb_share_fetch_session.adding_toppars)); + rd_list_destroy(rkb->rkb_share_fetch_session.adding_toppars); + rkb->rkb_share_fetch_session.adding_toppars = NULL; + } + + /* Clear forgetting toppars */ + if (rkb->rkb_share_fetch_session.forgetting_toppars) { + rd_rkb_dbg( + rkb, BROKER, "SHARESESSION", + "Clearing %d forgetting toppars from ShareFetch session on " + "clear", + rd_list_cnt( + rkb->rkb_share_fetch_session.forgetting_toppars)); + rd_list_destroy( + rkb->rkb_share_fetch_session.forgetting_toppars); + rkb->rkb_share_fetch_session.forgetting_toppars = NULL; + } + + /* + * This allows us to avoid future changes to the closed share session + * Toppar add/remove functions do not allow updates if epoch is -1 + * Avoid future changes to the closed share session */ + rkb->rkb_share_fetch_session.epoch = -1; +} + +void rd_kafka_broker_share_fetch_session_leave(rd_kafka_broker_t *rkb, + rd_kafka_op_t *rko_orig, + rd_ts_t now) { + rd_kafka_cgrp_t *rkcg = rkb->rkb_rk->rk_cgrp; + + if (rkb->rkb_share_fetch_session.epoch > 0) { + rd_rkb_dbg(rkb, BROKER, "SHARESESSION", + "Processing SHARE_FETCH op: " + "should_leave = true"); + + /* Set epoch to -1 to signal session close before sending + * request. This ensures session_update_epoch() skips + * incrementing on reply. */ + rkb->rkb_share_fetch_session.epoch = -1; + rd_kafka_ShareAcknowledgeRequest( + rkb, rkcg->rkcg_group_id, rkcg->rkcg_member_id, + -1, /* epoch=-1 signals session close */ + rko_orig, now); + } else { + rd_rkb_dbg(rkb, BROKER, "SHARESESSION", + "Ignoring SHARE_FETCH op with " + "should_leave = true: " + "no active session"); + + if (rkb->rkb_share_fetch_session.epoch == 0) + /* Required as it is possible that we were about + * to establish a session */ + rd_kafka_broker_share_fetch_session_clear(rkb); + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rko_orig, RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND); + } +} + +void rd_kafka_broker_share_rpc(rd_kafka_broker_t *rkb, + rd_kafka_op_t *rko_orig, + rd_ts_t now) { + + rd_kafka_cgrp_t *rkcg = rkb->rkb_rk->rk_cgrp; + int32_t max_records = 0; + rd_list_t *ack_details = rko_orig->rko_u.share_fetch.ack_details; + rd_bool_t has_ack_details = ack_details && rd_list_cnt(ack_details) > 0; + rd_list_t *toppars_to_add = NULL; + rd_list_t *toppars_to_forget = NULL; + rd_list_t *stripped_acks = NULL; + + if (!rko_orig->rko_u.share_fetch.should_fetch && !has_ack_details) { + rd_rkb_dbg(rkb, FETCH, "SHARERPC", + "Not sending Share RPC: " + "no fetch requested and no acknowledgements"); + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rko_orig, RD_KAFKA_RESP_ERR__NOOP); + return; + } + + if (!rkcg->rkcg_member_id) { + rd_rkb_dbg(rkb, FETCH, "SHARERPC", + "Share RPC requested without member_id"); + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rko_orig, RD_KAFKA_RESP_ERR__INVALID_ARG); + return; + } + + if (!rko_orig->rko_u.share_fetch.should_fetch) { + /* Ack-only: use ShareAcknowledge RPC. + * + * If session epoch is 0 (new consumer or post-reset), + * the broker has no session state to acknowledge + * against. Fail the acks locally with + * INVALID_SHARE_SESSION_EPOCH instead of sending, + * since the broker would reject the request. */ + if (rkb->rkb_share_fetch_session.epoch == 0) { + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "Failing %d ack batches locally: " + "session epoch is 0 (no session)", + rd_list_cnt(ack_details)); + rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rko_orig, + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH); + return; + } + + /* TODO KIP-932: add a defensive per-batch leader-stale + * strip here (mirroring the segregation-time check) to + * cover the window where the cached leader changes + * between segregation on the main thread and this RPC + * send on the broker thread. It must run AFTER the + * epoch==0 check above so the local-only failure path + * stays identical to the broker-roundtrip flow. */ + + rd_rkb_dbg(rkb, FETCH, "SHAREACK", + "Sending ShareAcknowledge Request with" + " acknowledgements"); + + rd_kafka_ShareAcknowledgeRequest( + rkb, rkcg->rkcg_group_id, rkcg->rkcg_member_id, + rkb->rkb_share_fetch_session.epoch, rko_orig, now); + return; + } + + max_records = rkb->rkb_rk->rk_conf.share.max_poll_records; + toppars_to_add = rkb->rkb_share_fetch_session.toppars_to_add; + toppars_to_forget = rkb->rkb_share_fetch_session.toppars_to_forget; + + /* If session epoch is 0 (new/reset session) and we have + * piggybacked acks, the broker has no session state to ack + * against. Fail the acks locally with + * INVALID_SHARE_SESSION_EPOCH and strip them from the wire + * request. The ShareFetch itself still goes out to establish + * the session. */ + if (rkb->rkb_share_fetch_session.epoch == 0 && has_ack_details) { + rd_kafka_share_ack_batches_t *batch; + int i; + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Stripping %d piggybacked ack batches: " + "session epoch is 0 (no session)", + rd_list_cnt(ack_details)); + + /* Pre-set each batch's err. Conditional buf-cb init + * preserves this; main reply handler propagates to + * commit_sync results / acknowledgement callback. */ + RD_LIST_FOREACH(batch, ack_details, i) { + batch->rktpar->err = + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH; + } + + /* Detach so ShareFetchRequest builds without the ack + * data section. Restored after the request is built + * so the buf reply handler / main thread reply + * handler can still read the per-batch err. */ + stripped_acks = rko_orig->rko_u.share_fetch.ack_details; + rko_orig->rko_u.share_fetch.ack_details = NULL; + has_ack_details = rd_false; + } + + /* TODO KIP-932: add a defensive per-batch leader-stale strip + * here (mirroring the segregation-time check) to cover the + * window where the cached leader changes between segregation + * on the main thread and this RPC send on the broker thread. + * It must run AFTER the epoch==0 strip above so the local-only + * failure path stays identical to the broker-roundtrip flow. */ + + rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", + "Sending ShareFetch Request (epoch %" PRId32 + ") with%s%s%s fetching messages", + rkb->rkb_share_fetch_session.epoch, + has_ack_details ? " acknowledgements," : "", + toppars_to_add ? " new topics," : "", + toppars_to_forget ? " forgotten toppars," : ""); + + rd_kafka_ShareFetchRequest( + rkb, rkcg->rkcg_group_id, /* group_id */ + rkcg->rkcg_member_id, /* member_id */ + rkb->rkb_share_fetch_session.epoch, /* share_session_epoch */ + rkb->rkb_rk->rk_conf.fetch_wait_max_ms, + rkb->rkb_rk->rk_conf.fetch_min_bytes, + rkb->rkb_rk->rk_conf.fetch_max_bytes, max_records, + max_records, /* batch_size: same value as max_records, + * both sourced from max.poll.records */ + toppars_to_add, /* toppars to add to session */ + toppars_to_forget, /* forgetting toppars */ + rko_orig, /* rko (carries ack_details) */ + now); + + /* Restore ack_details so the buf reply handler / main thread + * reply handler can read the pre-set per-batch err. */ + if (stripped_acks) + rko_orig->rko_u.share_fetch.ack_details = stripped_acks; +} + +/** + * @brief Build and send a Fetch request message for all underflowed toppars + * for a specific broker. + * + * @returns the number of partitions included in the FetchRequest, if any. + * + * @locality broker thread + */ +int rd_kafka_broker_fetch_toppars(rd_kafka_broker_t *rkb, rd_ts_t now) { + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + int cnt = 0; + size_t of_TopicArrayCnt = 0; + int TopicArrayCnt = 0; + size_t of_PartitionArrayCnt = 0; + int PartitionArrayCnt = 0; + rd_kafka_topic_t *rkt_last = NULL; + int16_t ApiVersion = 0; + + /* Create buffer and segments: + * 1 x ReplicaId MaxWaitTime MinBytes TopicArrayCnt + * N x topic name + * N x PartitionArrayCnt Partition FetchOffset MaxBytes + * where N = number of toppars. + * Since we dont keep track of the number of topics served by + * this broker, only the partition count, we do a worst-case calc + * when allocating and assume each partition is on its own topic + */ + + if (unlikely(rkb->rkb_active_toppar_cnt == 0)) + return 0; + + ApiVersion = rd_kafka_broker_ApiVersion_supported(rkb, RD_KAFKAP_Fetch, + 0, 16, NULL); + + /* Fallback to version 12 if topic id is null which can happen if + * inter.broker.protocol.version is < 2.8 */ + if (ApiVersion > 12 && !can_use_topic_ids(rkb)) + ApiVersion = 12; + + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_Fetch, 1, + /* MaxWaitTime+MinBytes+MaxBytes+IsolationLevel+ + * SessionId+Epoch+TopicCnt */ + 4 + 4 + 4 + 1 + 4 + 4 + 4 + + /* N x PartCnt+Partition+CurrentLeaderEpoch+FetchOffset+ + * LastFetchedEpoch+LogStartOffset+MaxBytes+?TopicNameLen?*/ + (rkb->rkb_active_toppar_cnt * + (4 + 4 + 4 + 8 + 4 + 8 + 4 + 40)) + + /* ForgottenTopicsCnt */ + 4 + + /* N x ForgottenTopicsData */ + 0, + ApiVersion >= 12); + + if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2) + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, + RD_KAFKA_FEATURE_MSGVER2); + else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1) + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, + RD_KAFKA_FEATURE_MSGVER1); + else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME) + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, + RD_KAFKA_FEATURE_THROTTLETIME); + + + /* FetchRequest header */ + if (rd_kafka_buf_ApiVersion(rkbuf) <= 14) + /* ReplicaId */ + rd_kafka_buf_write_i32(rkbuf, -1); + + /* MaxWaitTime */ + rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_wait_max_ms); + /* MinBytes */ + rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_min_bytes); + + if (rd_kafka_buf_ApiVersion(rkbuf) >= 3) + /* MaxBytes */ + rd_kafka_buf_write_i32(rkbuf, + rkb->rkb_rk->rk_conf.fetch_max_bytes); + + if (rd_kafka_buf_ApiVersion(rkbuf) >= 4) + /* IsolationLevel */ + rd_kafka_buf_write_i8(rkbuf, + rkb->rkb_rk->rk_conf.isolation_level); + + if (rd_kafka_buf_ApiVersion(rkbuf) >= 7) { + /* SessionId */ + rd_kafka_buf_write_i32(rkbuf, 0); + /* Epoch */ + rd_kafka_buf_write_i32(rkbuf, -1); + } + + /* Write zero TopicArrayCnt but store pointer for later update */ + of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf); + + /* Prepare map for storing the fetch version for each partition, + * this will later be checked in Fetch response to purge outdated + * responses (e.g., after a seek). */ + rkbuf->rkbuf_rktp_vers = + rd_list_new(0, (void *)rd_kafka_toppar_ver_destroy); + rd_list_prealloc_elems(rkbuf->rkbuf_rktp_vers, + sizeof(struct rd_kafka_toppar_ver), + rkb->rkb_active_toppar_cnt, 0); + + /* Round-robin start of the list. */ + rktp = rkb->rkb_active_toppar_next; + do { + struct rd_kafka_toppar_ver *tver; + + if (rkt_last != rktp->rktp_rkt) { + if (rkt_last != NULL) { + /* Update PartitionArrayCnt */ + rd_kafka_buf_finalize_arraycnt( + rkbuf, of_PartitionArrayCnt, + PartitionArrayCnt); + /* Topic tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + /* TODO: This is not thread safe as topic can + be recreated in which case topic id is + updated from the main thread and we are + sending topic id from broker thread.*/ + if (rd_kafka_buf_ApiVersion(rkbuf) > 12) { + /* Topic id must be non-zero here */ + rd_dassert(!RD_KAFKA_UUID_IS_ZERO( + rktp->rktp_rkt->rkt_topic_id)); + /* Topic ID */ + rd_kafka_buf_write_uuid( + rkbuf, &rktp->rktp_rkt->rkt_topic_id); + } else { + /* Topic name */ + rd_kafka_buf_write_kstr( + rkbuf, rktp->rktp_rkt->rkt_topic); + } + + TopicArrayCnt++; + rkt_last = rktp->rktp_rkt; + /* Partition count */ + of_PartitionArrayCnt = + rd_kafka_buf_write_arraycnt_pos(rkbuf); + PartitionArrayCnt = 0; + } + + PartitionArrayCnt++; + + /* Partition */ + rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition); + + if (rd_kafka_buf_ApiVersion(rkbuf) >= 9) { + /* CurrentLeaderEpoch */ + if (rktp->rktp_leader_epoch < 0 && + rd_kafka_has_reliable_leader_epochs(rkb)) { + /* If current leader epoch is set to -1 and + * the broker has reliable leader epochs, + * send 0 instead, so that epoch is checked + * and optionally metadata is refreshed. + * This can happen if metadata is read initially + * without an existing topic (see + * rd_kafka_topic_metadata_update2). + */ + rd_kafka_buf_write_i32(rkbuf, 0); + } else { + rd_kafka_buf_write_i32(rkbuf, + rktp->rktp_leader_epoch); + } + } + /* FetchOffset */ + rd_kafka_buf_write_i64(rkbuf, + rktp->rktp_offsets.fetch_pos.offset); + if (rd_kafka_buf_ApiVersion(rkbuf) >= 12) + /* LastFetchedEpoch - only used by follower replica */ + rd_kafka_buf_write_i32(rkbuf, -1); + if (rd_kafka_buf_ApiVersion(rkbuf) >= 5) + /* LogStartOffset - only used by follower replica */ + rd_kafka_buf_write_i64(rkbuf, -1); + + /* MaxBytes */ + rd_kafka_buf_write_i32(rkbuf, rktp->rktp_fetch_msg_max_bytes); + + /* Partition tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + + rd_rkb_dbg(rkb, FETCH, "FETCH", + "Fetch topic %.*s [%" PRId32 "] at offset %" PRId64 + " (leader epoch %" PRId32 + ", current leader epoch %" PRId32 ", v%d)", + RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic), + rktp->rktp_partition, + rktp->rktp_offsets.fetch_pos.offset, + rktp->rktp_offsets.fetch_pos.leader_epoch, + rktp->rktp_leader_epoch, rktp->rktp_fetch_version); + + /* We must have a valid fetch offset when we get here */ + rd_dassert(rktp->rktp_offsets.fetch_pos.offset >= 0); + + /* Add toppar + op version mapping. */ + tver = rd_list_add(rkbuf->rkbuf_rktp_vers, NULL); + tver->rktp = rd_kafka_toppar_keep(rktp); + tver->version = rktp->rktp_fetch_version; + + cnt++; + } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp, + rktp_activelink)) != + rkb->rkb_active_toppar_next); + + /* Update next toppar to fetch in round-robin list. */ + rd_kafka_broker_active_toppar_next( + rkb, rktp ? CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp, + rktp_activelink) + : NULL); + + rd_rkb_dbg(rkb, FETCH, "FETCH", "Fetch %i/%i/%i toppar(s)", cnt, + rkb->rkb_active_toppar_cnt, rkb->rkb_toppar_cnt); + if (!cnt) { + rd_kafka_buf_destroy(rkbuf); + return cnt; + } + + if (rkt_last != NULL) { + /* Update last topic's PartitionArrayCnt */ + rd_kafka_buf_finalize_arraycnt(rkbuf, of_PartitionArrayCnt, + PartitionArrayCnt); + /* Topic tags */ + rd_kafka_buf_write_tags_empty(rkbuf); + } + + /* Update TopicArrayCnt */ + rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt); + + + if (rd_kafka_buf_ApiVersion(rkbuf) >= 7) + /* Length of the ForgottenTopics list (KIP-227). Broker + * use only - not used by the consumer. */ + rd_kafka_buf_write_arraycnt(rkbuf, 0); + + if (rd_kafka_buf_ApiVersion(rkbuf) >= 11) + /* RackId */ + rd_kafka_buf_write_kstr(rkbuf, + rkb->rkb_rk->rk_conf.client_rack); + + /* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */ + if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000) + rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING; + + /* Use configured timeout */ + rd_kafka_buf_set_timeout(rkbuf, + rkb->rkb_rk->rk_conf.socket_timeout_ms + + rkb->rkb_rk->rk_conf.fetch_wait_max_ms, + now); + + /* Sort toppar versions for quicker lookups in Fetch response. */ + rd_list_sort(rkbuf->rkbuf_rktp_vers, rd_kafka_toppar_ver_cmp); + + rkb->rkb_fetching = 1; + rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_fetch_reply, NULL); + + return cnt; +} + +/** + * @brief Decide whether it should start fetching from next fetch start + * or continue with current fetch pos. + * + * @param rktp the toppar + * + * @returns rd_true if it should start fetching from next fetch start, + * rd_false otherwise. + * + * @locality any + * @locks toppar_lock() MUST be held + */ +static rd_bool_t rd_kafka_toppar_fetch_decide_start_from_next_fetch_start( + rd_kafka_toppar_t *rktp) { + return rktp->rktp_op_version > rktp->rktp_fetch_version || + rd_kafka_fetch_pos_cmp(&rktp->rktp_next_fetch_start, + &rktp->rktp_last_next_fetch_start) || + rktp->rktp_offsets.fetch_pos.offset == RD_KAFKA_OFFSET_INVALID; +} + +/** + * @brief Return next fetch start position: + * if it should start fetching from next fetch start + * or continue with current fetch pos. + * + * @param rktp The toppar + * + * @returns Next fetch start position + * + * @locality any + * @locks toppar_lock() MUST be held + */ +rd_kafka_fetch_pos_t +rd_kafka_toppar_fetch_decide_next_fetch_start_pos(rd_kafka_toppar_t *rktp) { + if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp)) + return rktp->rktp_next_fetch_start; + else + return rktp->rktp_offsets.fetch_pos; +} + +/** + * @brief Decide whether this toppar should be on the fetch list or not. + * + * Also: + * - update toppar's op version (for broker thread's copy) + * - finalize statistics (move rktp_offsets to rktp_offsets_fin) + * + * @returns the partition's Fetch backoff timestamp, or 0 if no backoff. + * + * @locality broker thread + * @locks none + */ +rd_ts_t rd_kafka_toppar_fetch_decide(rd_kafka_toppar_t *rktp, + rd_kafka_broker_t *rkb, + int force_remove) { + int should_fetch = 1; + const char *reason = ""; + int32_t version; + rd_ts_t ts_backoff = 0; + rd_bool_t lease_expired = rd_false; + + rd_kafka_toppar_lock(rktp); + + /* Check for preferred replica lease expiry */ + lease_expired = rktp->rktp_leader_id != rktp->rktp_broker_id && + rd_interval(&rktp->rktp_lease_intvl, + 5 * 60 * 1000 * 1000 /*5 minutes*/, 0) > 0; + if (lease_expired) { + /* delegate_to_leader() requires no locks to be held */ + rd_kafka_toppar_unlock(rktp); + rd_kafka_toppar_delegate_to_leader(rktp); + rd_kafka_toppar_lock(rktp); + + reason = "preferred replica lease expired"; + should_fetch = 0; + goto done; + } + + /* Forced removal from fetch list */ + if (unlikely(force_remove)) { + reason = "forced removal"; + should_fetch = 0; + goto done; + } + + if (unlikely((rktp->rktp_flags & RD_KAFKA_TOPPAR_F_REMOVE) != 0)) { + reason = "partition removed"; + should_fetch = 0; + goto done; + } + + /* Skip toppars not in active fetch state */ + if (rktp->rktp_fetch_state != RD_KAFKA_TOPPAR_FETCH_ACTIVE) { + reason = "not in active fetch state"; + should_fetch = 0; + goto done; + } + + /* Update broker thread's fetch op version */ + version = rktp->rktp_op_version; + if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp)) { + /* New version barrier, something was modified from the + * control plane. Reset and start over. + * Alternatively only the next_offset changed but not the + * barrier, which is the case when automatically triggering + * offset.reset (such as on PARTITION_EOF or + * OFFSET_OUT_OF_RANGE). */ + + rd_kafka_dbg( + rktp->rktp_rkt->rkt_rk, TOPIC, "FETCHDEC", + "Topic %s [%" PRId32 + "]: fetch decide: " + "updating to version %d (was %d) at %s " + "(was %s)", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + version, rktp->rktp_fetch_version, + rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start), + rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos)); + + rd_kafka_offset_stats_reset(&rktp->rktp_offsets); + + /* New start offset */ + rktp->rktp_offsets.fetch_pos = rktp->rktp_next_fetch_start; + rktp->rktp_last_next_fetch_start = rktp->rktp_next_fetch_start; + + rktp->rktp_fetch_version = version; + + /* Clear last error to propagate new fetch + * errors if encountered. */ + rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR; + + rd_kafka_q_purge_toppar_version(rktp->rktp_fetchq, rktp, + version); + } + + + if (RD_KAFKA_TOPPAR_IS_PAUSED(rktp)) { + should_fetch = 0; + reason = "paused"; + + } else if (RD_KAFKA_OFFSET_IS_LOGICAL( + rktp->rktp_next_fetch_start.offset)) { + should_fetch = 0; + reason = "no concrete offset"; + } else if (rktp->rktp_ts_fetch_backoff > rd_clock()) { + reason = "fetch backed off"; + ts_backoff = rktp->rktp_ts_fetch_backoff; + should_fetch = 0; + } else if (rd_kafka_q_len(rktp->rktp_fetchq) >= + rkb->rkb_rk->rk_conf.queued_min_msgs) { + /* Skip toppars who's local message queue is already above + * the lower threshold. */ + reason = "queued.min.messages exceeded"; + ts_backoff = rd_kafka_toppar_fetch_backoff( + rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL); + should_fetch = 0; + + } else if ((int64_t)rd_kafka_q_size(rktp->rktp_fetchq) >= + rkb->rkb_rk->rk_conf.queued_max_msg_bytes) { + reason = "queued.max.messages.kbytes exceeded"; + ts_backoff = rd_kafka_toppar_fetch_backoff( + rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL); + should_fetch = 0; + } + +done: + /* Copy offset stats to finalized place holder. */ + rktp->rktp_offsets_fin = rktp->rktp_offsets; + + if (rktp->rktp_fetch != should_fetch) { + rd_rkb_dbg( + rkb, FETCH, "FETCH", + "Topic %s [%" PRId32 + "] in state %s at %s " + "(%d/%d msgs, %" PRId64 + "/%d kb queued, " + "opv %" PRId32 ") is %s%s", + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + rd_kafka_fetch_states[rktp->rktp_fetch_state], + rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start), + rd_kafka_q_len(rktp->rktp_fetchq), + rkb->rkb_rk->rk_conf.queued_min_msgs, + rd_kafka_q_size(rktp->rktp_fetchq) / 1024, + rkb->rkb_rk->rk_conf.queued_max_msg_kbytes, + rktp->rktp_fetch_version, + should_fetch ? "fetchable" : "not fetchable: ", reason); + + if (should_fetch) { + rd_dassert(rktp->rktp_fetch_version > 0); + rd_kafka_broker_active_toppar_add( + rkb, rktp, *reason ? reason : "fetchable"); + } else { + rd_kafka_broker_active_toppar_del(rkb, rktp, reason); + } + } + + rd_kafka_toppar_unlock(rktp); + + /* Non-fetching partitions will have an + * indefinate backoff, unless explicitly specified. */ + if (!should_fetch && !ts_backoff) + ts_backoff = RD_TS_MAX; + + return ts_backoff; +} diff --git a/lib/librdkafka-2.10.1/src/rdkafka_fetcher.h b/lib/librdkafka-2.15.0/src/rdkafka_fetcher.h similarity index 69% rename from lib/librdkafka-2.10.1/src/rdkafka_fetcher.h rename to lib/librdkafka-2.15.0/src/rdkafka_fetcher.h index e304f1369f6..89a7784a598 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_fetcher.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_fetcher.h @@ -40,5 +40,23 @@ rd_ts_t rd_kafka_toppar_fetch_decide(rd_kafka_toppar_t *rktp, rd_kafka_broker_t *rkb, int force_remove); +void rd_kafka_broker_share_fetch_session_leave(rd_kafka_broker_t *rkb, + rd_kafka_op_t *rko_orig, + rd_ts_t now); +void rd_kafka_broker_share_fetch_session_clear(rd_kafka_broker_t *rkb); +void rd_kafka_broker_session_remove_partition_from_toppars_in_session( + rd_kafka_broker_t *rkb, + rd_kafka_toppar_t *rktp); +void rd_kafka_broker_share_rpc(rd_kafka_broker_t *rkb, + rd_kafka_op_t *rko_orig, + rd_ts_t now); +void rd_kafka_share_filter_acquired_records_and_update_ack_type( + rd_kafka_q_t *temp_fetchq, + rd_list_t *filtered_msgs, + const int64_t *FirstOffsets, + const int64_t *LastOffsets, + const int16_t *DeliveryCounts, + int32_t AcquiredRecordsArrayCnt); + #endif /* _RDKAFKA_FETCHER_H_ */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_header.c b/lib/librdkafka-2.15.0/src/rdkafka_header.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_header.c rename to lib/librdkafka-2.15.0/src/rdkafka_header.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_header.h b/lib/librdkafka-2.15.0/src/rdkafka_header.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_header.h rename to lib/librdkafka-2.15.0/src/rdkafka_header.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_idempotence.c b/lib/librdkafka-2.15.0/src/rdkafka_idempotence.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_idempotence.c rename to lib/librdkafka-2.15.0/src/rdkafka_idempotence.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_idempotence.h b/lib/librdkafka-2.15.0/src/rdkafka_idempotence.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_idempotence.h rename to lib/librdkafka-2.15.0/src/rdkafka_idempotence.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_int.h b/lib/librdkafka-2.15.0/src/rdkafka_int.h similarity index 79% rename from lib/librdkafka-2.10.1/src/rdkafka_int.h rename to lib/librdkafka-2.15.0/src/rdkafka_int.h index 6580afd1bbc..e2ed5d1dc63 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_int.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_int.h @@ -94,6 +94,13 @@ typedef struct rd_kafka_lwtopic_s rd_kafka_lwtopic_t; #define RD_KAFKA_OFFSET_IS_LOGICAL(OFF) ((OFF) < 0) +/** + * TODO KIP-932: Remove the check of RD_KAFKA_CONSUMER. Only + * checking for is_share_consumer should be sufficient. + */ +#define RD_KAFKA_IS_SHARE_CONSUMER(rk) \ + ((rk)->rk_type == RD_KAFKA_CONSUMER && \ + (rk)->rk_conf.share.is_share_consumer) /** * @struct Represents a fetch position: @@ -299,6 +306,10 @@ struct rd_kafka_s { * that have had at least one connection attempt * and are configured or learned. */ rd_atomic32_t rk_broker_down_cnt; + /** Set to 1 when there's a re-bootstrap in progress. + * Set to 0 when the re-bootstrap is done. + * Accessed from the main thread and the broker threads. */ + rd_atomic32_t rk_rebootstrap_in_progress; /**< Additional bootstrap servers list. * contains all brokers added through rd_kafka_brokers_add(). @@ -327,6 +338,78 @@ struct rd_kafka_s { struct rd_kafka_cgrp_s *rk_cgrp; + rd_kafka_share_t + *rk_rkshare; /**< Share consumer handle (if share + * consumer), NULL otherwise. + * Set when rd_kafka_share_consumer_new() + * is called, used to access inflight + * acks for building ack batches. */ + + /** Share consumer state. Only used when this rd_kafka_t handle + * is a share consumer (rk_rkshare != NULL). */ + /** + * TODO KIP-932: Change to rk_share when interface is finalized. + */ + struct { + /** Runtime acknowledgement callback set via + * rd_kafka_share_set_acknowledgement_commit_cb(). + * @locality APP THREAD ONLY (set in + * set_share_acknowledgement_commit_cb). + */ + void (*share_acknowledgement_commit_cb)( + rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque); + + /** Application opaque for acknowledgement callback. + * @locality APP THREAD ONLY. */ + void *acknowledgement_commit_cb_opaque; + + /** Reentrancy protection flag - set to true when inside + * callback. + * @locality APP THREAD ONLY. */ + rd_bool_t in_callback; + + /** Registration flag - whether callback is currently + * registered. + * Written via RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER op + * handler. Used by main thread to decide whether to + * enqueue callback ops. + * @locality MAIN THREAD ONLY. */ + rd_bool_t acknowledgement_commit_cb_registered; + + /** Single-thread access gate with re-entrant ownership. + * + * Every public share consumer API calls + * rd_kafka_share_acquire() on entry and + * rd_kafka_share_release() before returning. + * The owning thread is recorded in + * @c current_owner_thread; concurrent entry from a + * different thread is rejected with + * RD_KAFKA_RESP_ERR__CONFLICT. The owning thread may + * re-enter (e.g. nested share calls), with @c ref_count + * tracking the nesting level — ownership is released + * only when @c ref_count returns to 0. + * + * @c current_owner_thread holds a real per-thread + * identifier from thrd_current_id() + * (GetCurrentThreadId() on Windows, pthread_self() + * elsewhere); the unowned sentinel is + * RD_KAFKA_SHARE_NO_CURRENT_THREAD (-1) — a value no real + * thread id can take on any supported platform. 0 is + * deliberately *not* the sentinel so a missing init + * doesn't silently look "unowned" (a zero-initialised + * struct will fail acquire with __CONFLICT instead of + * letting two threads both pass through). The acquire + * path uses rd_atomic64_cas() so the + * first-caller-wins transition is race-free. */ + rd_atomic64_t current_owner_thread; /**< Owning thread id; -1 = + unowned. */ + rd_atomic32_t + ref_count; /**< Re-entrancy count for the owner. */ + } rk_share_consumer; + rd_kafka_conf_t rk_conf; rd_kafka_q_t *rk_logq; /* Log queue if `log.queue` set */ char rk_name[128]; @@ -392,16 +475,7 @@ struct rd_kafka_s { * to avoid reaching * max.poll.interval.ms. during that time * frame. Only relevant for consumer. */ - rd_ts_t rk_ts_last_poll_start; /**< Timestamp of last application - * consumer_poll() call start - * Only relevant for consumer. - * Not an atomic as Kafka consumer - * isn't thread safe. */ - rd_ts_t rk_ts_last_poll_end; /**< Timestamp of last application - * consumer_poll() call end - * Only relevant for consumer. - * Not an atomic as Kafka consumer - * isn't thread safe. */ + /* First fatal error. */ struct { rd_atomic32_t err; /**< rd_kafka_resp_err_t */ @@ -737,6 +811,15 @@ struct rd_kafka_s { * started */ /** Total rebalance latency (ms) up to previous push */ uint64_t rebalance_latency_total; + /** Total share fetch requests up to previous push */ + int64_t share_fetch_total; + /** Total share bytes consumed up to previous push */ + int64_t share_bytes_consumed_total; + /** Total acknowledgements sent up to previous push */ + int64_t acknowledgements_send_total; + /** Total ShareGroupHeartbeat responses up to previous + * push */ + int64_t heartbeat_total; } rk_historic_c; struct { @@ -746,6 +829,12 @@ struct rd_kafka_s { rd_avg_t rk_avg_rebalance_latency; /**< Current rebalance * latency avg */ + rd_avg_t rk_avg_share_poll_idle_ratio; + rd_avg_t + rk_avg_share_time_between_poll; /**< Current time + between two + share_consume_batch + */ } rd_avg_current; struct { @@ -755,8 +844,32 @@ struct rd_kafka_s { rd_avg_t rk_avg_rebalance_latency; /**< Rolled over rebalance * latency avg */ + rd_avg_t rk_avg_share_poll_idle_ratio; + rd_avg_t + rk_avg_share_time_between_poll; /**< Rolled over + time between two + share_consume_batch + */ } rd_avg_rollover; + /* Share consumer poll/batch tracking */ + struct { + rd_ts_t ts_last_poll_start; + rd_ts_t ts_poll_start; + rd_ts_t time_since_last_poll; + } rk_share_poll; + + /* Share consumer fetch-RPC counter */ + rd_atomic64_t share_fetch_total; + + /* Share consumer total bytes consumed */ + rd_atomic64_t share_bytes_consumed_total; + + /* Share consumer record-level acknowledgements sent counter */ + rd_atomic64_t acknowledgements_send_total; + + /* Share consumer ShareGroupHeartbeat response counter */ + rd_atomic64_t heartbeat_total; } rk_telemetry; /* Test mocks */ @@ -774,6 +887,70 @@ struct rd_kafka_s { } rk_mock; }; +/** + * @brief Map type for tracking inflight share acknowledgements. + * Key: topic-partition, Value: ack batches for that partition. + */ +typedef RD_MAP_TYPE(const rd_kafka_topic_partition_t *, + rd_kafka_share_ack_batches_t *) + rd_kafka_share_inflight_map_t; + +struct rd_kafka_share_s { + rd_kafka_t *rkshare_rk; /**< The shared rd_kafka_t instance */ + + /** + * TODO KIP-932: Decide whether we want to keep this here + * or move it in rd_kafka_t. + */ + /** Map of inflight acknowledgements per topic-partition. + * Key: rd_kafka_topic_partition_t* + * Value: rd_kafka_share_ack_batches_t* + */ + rd_kafka_share_inflight_map_t rkshare_inflight_acks; + + /** Total number of unacknowledged messages across all partitions. */ + int64_t rkshare_unacked_cnt; + + /** Set when a FANOUT with fetch_more_records=true is sent. + * Reset when records are received in the consumer queue. + * Prevents duplicate fetch FANOUTs while one is in flight. */ + rd_bool_t rkshare_fetch_more_records_requested; + + /** + * Set to true when the consumer is closing. Will be used to: + * * prevent any new API call from the app. + * * check if SHARE_FETCH_RESPONSE is received to poll_cb only + * while closing + */ + rd_bool_t rkshare_consumer_closing; + + /** True when the consumer has an active subscription (i.e. + * rd_kafka_share_subscribe() succeeded with a non-empty list + * and rd_kafka_share_unsubscribe() has not been called since). + * Written only on the app thread under rkshare_acquire/release; + * read only in consume_batch while the same lock is held. + * @locality app thread */ + rd_bool_t rkshare_subscribed; +}; + +/** + * @brief Opaque message list returned by rd_kafka_share_poll(). + * + * The struct is allocated as a flexible array sized to the exact message + * count returned by a single poll. The caller releases the list and all + * contained messages via rd_kafka_messages_destroy(). + */ +/** + * TODO KIP-932: GA: Fix this to use FAM with message struct not + * pointer to messages. + */ +struct rd_kafka_messages_s { + size_t cnt; /**< Number of messages. */ + rd_kafka_message_t *elems[]; /**< Message pointer array. */ +}; + +rd_kafka_messages_t *rd_kafka_messages_new(size_t cnt); + #define rd_kafka_wrlock(rk) rwlock_wrlock(&(rk)->rk_lock) #define rd_kafka_rdlock(rk) rwlock_rdlock(&(rk)->rk_lock) #define rd_kafka_rdunlock(rk) rwlock_rdunlock(&(rk)->rk_lock) @@ -1097,6 +1274,14 @@ int rd_kafka_set_fatal_error0(rd_kafka_t *rk, rd_kafka_error_t *rd_kafka_get_fatal_error(rd_kafka_t *rk); +#define rd_kafka_producer_can_have_fatal_errors(rk) \ + (rk->rk_type == RD_KAFKA_PRODUCER && rk->rk_conf.eos.idempotence) + +#define rd_kafka_consumer_can_have_fatal_errors(rk) \ + (rk->rk_type == RD_KAFKA_CONSUMER && \ + (rk->rk_conf.group_instance_id || \ + rk->rk_conf.group_protocol == RD_KAFKA_GROUP_PROTOCOL_CONSUMER)) + static RD_INLINE RD_UNUSED rd_kafka_resp_err_t rd_kafka_fatal_error_code(rd_kafka_t *rk) { /* This is an optimization to avoid an atomic read which are costly @@ -1106,10 +1291,8 @@ rd_kafka_fatal_error_code(rd_kafka_t *rk) { * 2) static consumers (group.instance.id) * 3) Group using consumer protocol (Introduced in KIP-848). See exact * errors in rd_kafka_cgrp_handle_ConsumerGroupHeartbeat() */ - if ((rk->rk_type == RD_KAFKA_PRODUCER && rk->rk_conf.eos.idempotence) || - (rk->rk_type == RD_KAFKA_CONSUMER && - (rk->rk_conf.group_instance_id || - rk->rk_conf.group_protocol == RD_KAFKA_GROUP_PROTOCOL_CONSUMER))) + if (rd_kafka_producer_can_have_fatal_errors(rk) || + rd_kafka_consumer_can_have_fatal_errors(rk)) return rd_atomic32_get(&rk->rk_fatal.err); return RD_KAFKA_RESP_ERR_NO_ERROR; @@ -1121,6 +1304,7 @@ extern char RD_TLS rd_kafka_thread_name[64]; void rd_kafka_set_thread_name(const char *fmt, ...) RD_FORMAT(printf, 1, 2); void rd_kafka_set_thread_sysname(const char *fmt, ...) RD_FORMAT(printf, 1, 2); +void rd_kafka_thread_srand(rd_kafka_t *rk, rd_bool_t internal_thread); int rd_kafka_path_is_dir(const char *path); rd_bool_t rd_kafka_dir_is_empty(const char *path); @@ -1179,29 +1363,31 @@ static RD_INLINE RD_UNUSED int rd_kafka_max_poll_exceeded(rd_kafka_t *rk) { * @locality any * @locks none */ -static RD_INLINE RD_UNUSED void -rd_kafka_app_poll_start(rd_kafka_t *rk, rd_ts_t now, rd_bool_t is_blocking) { - if (rk->rk_type != RD_KAFKA_CONSUMER) +static RD_INLINE RD_UNUSED void rd_kafka_app_poll_start(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_ts_t now, + rd_bool_t is_blocking) { + if (rk->rk_type != RD_KAFKA_CONSUMER || RD_KAFKA_IS_SHARE_CONSUMER(rk)) return; if (!now) now = rd_clock(); if (is_blocking) rd_atomic64_set(&rk->rk_ts_last_poll, INT64_MAX); - if (rk->rk_ts_last_poll_end) { + if (rkq->rkq_ts_last_poll_end) { int64_t poll_idle_ratio = 0; - rd_ts_t poll_interval = now - rk->rk_ts_last_poll_start; + rd_ts_t poll_interval = now - rkq->rkq_ts_last_poll_start; if (poll_interval) { - rd_ts_t idle_interval = - rk->rk_ts_last_poll_end - rk->rk_ts_last_poll_start; + rd_ts_t idle_interval = rkq->rkq_ts_last_poll_end - + rkq->rkq_ts_last_poll_start; poll_idle_ratio = idle_interval * 1000000 / poll_interval; } rd_avg_add( &rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio, poll_idle_ratio); - rk->rk_ts_last_poll_start = now; - rk->rk_ts_last_poll_end = 0; + rkq->rkq_ts_last_poll_start = now; + rkq->rkq_ts_last_poll_end = 0; } } @@ -1213,8 +1399,10 @@ rd_kafka_app_poll_start(rd_kafka_t *rk, rd_ts_t now, rd_bool_t is_blocking) { * @locality any * @locks none */ -static RD_INLINE RD_UNUSED void rd_kafka_app_polled(rd_kafka_t *rk) { - if (rk->rk_type == RD_KAFKA_CONSUMER) { +static RD_INLINE RD_UNUSED void rd_kafka_app_polled(rd_kafka_t *rk, + rd_kafka_q_t *rkq) { + if (rk->rk_type == RD_KAFKA_CONSUMER && + !RD_KAFKA_IS_SHARE_CONSUMER(rk)) { rd_ts_t now = rd_clock(); rd_atomic64_set(&rk->rk_ts_last_poll, now); if (unlikely(rk->rk_cgrp && @@ -1226,11 +1414,10 @@ static RD_INLINE RD_UNUSED void rd_kafka_app_polled(rd_kafka_t *rk) { rk->rk_cgrp, "app polled after poll interval exceeded"); } - if (!rk->rk_ts_last_poll_end) - rk->rk_ts_last_poll_end = now; - /* Disabled until #5017 is merged - * rd_dassert(rk->rk_ts_last_poll_end >= - * rk->rk_ts_last_poll_start); */ + if (!rkq->rkq_ts_last_poll_end) + rkq->rkq_ts_last_poll_end = now; + rd_dassert(rkq->rkq_ts_last_poll_end >= + rkq->rkq_ts_last_poll_start); } } @@ -1248,5 +1435,71 @@ rd_kafka_resp_err_t rd_kafka_background_thread_create(rd_kafka_t *rk, void rd_kafka_rebootstrap(rd_kafka_t *rk); +void rd_kafka_rebootstrap_tmr_start_maybe(rd_kafka_t *rk); + +int rd_kafka_rebootstrap_tmr_stop(rd_kafka_t *rk); + +void rd_kafka_reset_any_broker_down_reported(rd_kafka_t *rk); + +rd_kafka_op_res_t rd_kafka_share_fetch_fanout_op(rd_kafka_t *rk, + rd_kafka_q_t *rkq, + rd_kafka_op_t *rko); + +/** + * @brief Return an error object if the share consumer is closed or closing. + * @returns a newly-allocated error (owned by caller) or NULL if usable. + */ +rd_kafka_error_t * +rd_kafka_share_consumer_closed_error(rd_kafka_share_t *rkshare); + +/** + * @brief Return a response-error code if the share consumer is closed or + * closing. + * @returns RD_KAFKA_RESP_ERR__STATE if closed/closing, else + * RD_KAFKA_RESP_ERR_NO_ERROR. + */ +rd_kafka_resp_err_t +rd_kafka_share_consumer_closed_err(rd_kafka_share_t *rkshare); + +/** + * @brief Sentinel value stored in rk_share_consumer.current_owner_thread + * when the gate is unowned. Chosen to never collide with a real + * thread id on any supported platform. A zero-initialised struct + * does NOT look unowned, which is intentional: a missing + * init causes acquire to fail with __CONFLICT rather than + * silently letting two threads pass. + */ +#define RD_KAFKA_SHARE_NO_CURRENT_THREAD ((int64_t)(-1)) + +/** + * @brief Acquire single-thread ownership of the share consumer. + * + * Every public share consumer API must call this on entry and pair + * it with rd_kafka_share_release() on every return path. The owning + * thread may re-enter (nested share calls bump the ref_count counter + * and release decrements it). + * + * @returns NULL on success, or a newly-allocated rd_kafka_error_t (owned + * by the caller) on rejection: + * - RD_KAFKA_RESP_ERR__STATE on NULL rkshare, or when called + * from inside the user-defined acknowledgement commit + * callback. + * - RD_KAFKA_RESP_ERR__CONFLICT on concurrent entry from a + * different application thread. + */ +rd_kafka_error_t *rd_kafka_share_acquire(rd_kafka_share_t *rkshare); + +/** + * @brief Release ownership acquired by rd_kafka_share_acquire(). + * + * Tolerates NULL rkshare (no-op) so error paths can call it + * unconditionally. + */ +void rd_kafka_share_release(rd_kafka_share_t *rkshare); + +void rd_kafka_share_enqueue_fetch_op(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + rd_bool_t should_fetch, + rd_bool_t should_leave); #endif /* _RDKAFKA_INT_H_ */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_interceptor.c b/lib/librdkafka-2.15.0/src/rdkafka_interceptor.c similarity index 95% rename from lib/librdkafka-2.10.1/src/rdkafka_interceptor.c rename to lib/librdkafka-2.15.0/src/rdkafka_interceptor.c index b5bacece3cb..3c8438c938c 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_interceptor.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_interceptor.c @@ -701,12 +701,26 @@ rd_kafka_conf_interceptor_add_on_new(rd_kafka_conf_t *conf, } +/* Interceptors are not supported for share consumers. The hooks are + * either share-incompatible (on_consume fires only on the regular fetch + * path; on_commit is tied to offset commits which share consumers do not + * perform) or out of scope for the share consumer, and the Java share + * consumer rejects interceptors (ConsumerInterceptor) outright. All + * handle-level interceptor registrations are therefore rejected for share + * consumers. */ +#define RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk) \ + do { \ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) \ + return RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED; \ + } while (0) + rd_kafka_resp_err_t rd_kafka_interceptor_add_on_destroy( rd_kafka_t *rk, const char *ic_name, rd_kafka_interceptor_f_on_destroy_t *on_destroy, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_destroy, ic_name, (void *)on_destroy, ic_opaque); @@ -718,6 +732,7 @@ rd_kafka_interceptor_add_on_send(rd_kafka_t *rk, rd_kafka_interceptor_f_on_send_t *on_send, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_send, ic_name, (void *)on_send, ic_opaque); @@ -729,6 +744,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_acknowledgement( rd_kafka_interceptor_f_on_acknowledgement_t *on_acknowledgement, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_acknowledgement, ic_name, (void *)on_acknowledgement, ic_opaque); @@ -741,6 +757,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_consume( rd_kafka_interceptor_f_on_consume_t *on_consume, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_consume, ic_name, (void *)on_consume, ic_opaque); @@ -753,6 +770,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_commit( rd_kafka_interceptor_f_on_commit_t *on_commit, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_commit, ic_name, (void *)on_commit, ic_opaque); @@ -765,6 +783,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_request_sent( rd_kafka_interceptor_f_on_request_sent_t *on_request_sent, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_request_sent, ic_name, (void *)on_request_sent, ic_opaque); @@ -777,6 +796,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_response_received( rd_kafka_interceptor_f_on_response_received_t *on_response_received, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_response_received, ic_name, (void *)on_response_received, ic_opaque); @@ -789,6 +809,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_thread_start( rd_kafka_interceptor_f_on_thread_start_t *on_thread_start, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_thread_start, ic_name, (void *)on_thread_start, ic_opaque); @@ -801,6 +822,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_thread_exit( rd_kafka_interceptor_f_on_thread_exit_t *on_thread_exit, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_thread_exit, ic_name, (void *)on_thread_exit, ic_opaque); @@ -813,6 +835,7 @@ rd_kafka_resp_err_t rd_kafka_interceptor_add_on_broker_state_change( rd_kafka_interceptor_f_on_broker_state_change_t *on_broker_state_change, void *ic_opaque) { assert(!rk->rk_initialized); + RD_KAFKA_INTERCEPTOR_REJECT_IF_SHARE(rk); return rd_kafka_interceptor_method_add( &rk->rk_conf.interceptors.on_broker_state_change, ic_name, (void *)on_broker_state_change, ic_opaque); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_interceptor.h b/lib/librdkafka-2.15.0/src/rdkafka_interceptor.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_interceptor.h rename to lib/librdkafka-2.15.0/src/rdkafka_interceptor.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_lz4.c b/lib/librdkafka-2.15.0/src/rdkafka_lz4.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_lz4.c rename to lib/librdkafka-2.15.0/src/rdkafka_lz4.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_lz4.h b/lib/librdkafka-2.15.0/src/rdkafka_lz4.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_lz4.h rename to lib/librdkafka-2.15.0/src/rdkafka_lz4.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_metadata.c b/lib/librdkafka-2.15.0/src/rdkafka_metadata.c similarity index 91% rename from lib/librdkafka-2.10.1/src/rdkafka_metadata.c rename to lib/librdkafka-2.15.0/src/rdkafka_metadata.c index ab229bebe46..fe6572223ee 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_metadata.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_metadata.c @@ -375,8 +375,9 @@ static void rd_kafka_parse_Metadata_update_topic( rd_rkb_dbg(rkb, METADATA, "METADATA", /* The indent below is intentional */ - " Topic %s with %i partitions%s%s", mdt->topic, - mdt->partition_cnt, mdt->err ? ": " : "", + " Topic %s with %i partitions%s%s", + rd_kafka_topic_name_str_safe(mdt->topic), mdt->partition_cnt, + mdt->err ? ": " : "", mdt->err ? rd_kafka_err2str(mdt->err) : ""); /* Ignore metadata completely for temporary errors. (issue #513) @@ -387,8 +388,8 @@ static void rd_kafka_parse_Metadata_update_topic( rd_rkb_dbg(rkb, TOPIC, "METADATA", "Temporary error in metadata reply for " "topic %s (PartCnt %i): %s: ignoring", - mdt->topic, mdt->partition_cnt, - rd_kafka_err2str(mdt->err)); + rd_kafka_topic_name_str_safe(mdt->topic), + mdt->partition_cnt, rd_kafka_err2str(mdt->err)); } else { /* Update local topic & partition state based * on metadata */ @@ -407,13 +408,7 @@ static void rd_kafka_parse_Metadata_update_topic( * @locality rdkafka main thread */ rd_bool_t rd_kafka_has_reliable_leader_epochs(rd_kafka_broker_t *rkb) { - int features; - int16_t ApiVersion = 0; - - ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_Metadata, 0, 9, &features); - - return ApiVersion >= 9; + return rd_kafka_broker_ApiVersion_at_least(rkb, RD_KAFKAP_Metadata, 9); } /* Populates the topic partition to rack mapping for the the topic given by @@ -572,6 +567,7 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, int broker_changes = 0; int cache_changes = 0; int cgrp_subscription_version = -1; + int16_t ErrorCode = 0; /* If client rack is present, the metadata cache (topic or full) needs * to contain the partition to rack map. */ @@ -740,20 +736,22 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, if (!(md->topics[i].partitions = rd_tmpabuf_alloc( &tbuf, md->topics[i].partition_cnt * sizeof(*md->topics[i].partitions)))) - rd_kafka_buf_parse_fail(rkbuf, - "%s: %d partitions: " - "tmpabuf memory shortage", - md->topics[i].topic, - md->topics[i].partition_cnt); + rd_kafka_buf_parse_fail( + rkbuf, + "%s: %d partitions: " + "tmpabuf memory shortage", + rd_kafka_topic_name_str_safe(md->topics[i].topic), + md->topics[i].partition_cnt); if (!(mdi->topics[i].partitions = rd_tmpabuf_alloc( &tbuf, md->topics[i].partition_cnt * sizeof(*mdi->topics[i].partitions)))) - rd_kafka_buf_parse_fail(rkbuf, - "%s: %d internal partitions: " - "tmpabuf memory shortage", - md->topics[i].topic, - md->topics[i].partition_cnt); + rd_kafka_buf_parse_fail( + rkbuf, + "%s: %d internal partitions: " + "tmpabuf memory shortage", + rd_kafka_topic_name_str_safe(md->topics[i].topic), + md->topics[i].partition_cnt); for (j = 0; j < md->topics[i].partition_cnt; j++) { @@ -797,7 +795,8 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, "%s [%" PRId32 "]: %d replicas: " "tmpabuf memory shortage", - md->topics[i].topic, + rd_kafka_topic_name_str_safe( + md->topics[i].topic), md->topics[i].partitions[j].id, md->topics[i].partitions[j].replica_cnt); @@ -825,7 +824,8 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, "%s [%" PRId32 "]: %d isrs: " "tmpabuf memory shortage", - md->topics[i].topic, + rd_kafka_topic_name_str_safe( + md->topics[i].topic), md->topics[i].partitions[j].id, md->topics[i].partitions[j].isr_cnt); @@ -872,8 +872,21 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, ClusterAuthorizedOperations; } + if (ApiVersion >= 13) { + rd_kafka_buf_read_i16(rkbuf, &ErrorCode); + } + rd_kafka_buf_skip_tags(rkbuf); + if (ErrorCode) { + rd_rkb_dbg(rkb, METADATA, "METADATA", + "Metadata response: received top level " + "error code %" PRId16 ": %s", + ErrorCode, rd_kafka_err2str(ErrorCode)); + err = ErrorCode; + goto err; + } + /* Entire Metadata response now parsed without errors: * update our internal state according to the response. */ @@ -898,8 +911,13 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, for (i = 0; i < md->topic_cnt; i++) { - /* Ignore topics in blacklist */ + /* Ignore topics in blacklist. Skip when the broker + * returned a NULL topic name (e.g. UNKNOWN_TOPIC_ID + * response from a by-id Metadata request) — the + * blacklist matches on name only, so there is nothing + * to compare against. */ if (rkb->rkb_rk->rk_conf.topic_blacklist && + md->topics[i].topic && rd_kafka_pattern_match(rkb->rkb_rk->rk_conf.topic_blacklist, md->topics[i].topic)) { rd_rkb_dbg(rkb, TOPIC | RD_KAFKA_DBG_METADATA, @@ -925,7 +943,11 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, rd_kafka_parse_Metadata_update_topic(rkb, &md->topics[i], &mdi->topics[i]); - if (requested_topics) + /* Skip the by-name dedup if the response carries a + * NULL topic name; strcmp(NULL, ...) is UB. The + * missing_topics list is keyed by name, so a + * NULL-name response cannot match any entry. */ + if (requested_topics && md->topics[i].topic) rd_list_free_cb(missing_topics, rd_list_remove_cmp(missing_topics, md->topics[i].topic, @@ -1102,6 +1124,9 @@ rd_kafka_parse_Metadata0(rd_kafka_broker_t *rkb, rd_kafka_cgrp_metadata_update_check(rkb->rkb_rk->rk_cgrp, rd_true /*do join*/); + if (rk->rk_cgrp && RD_KAFKA_IS_SHARE_CONSUMER(rk)) + rd_kafka_share_topic_err_propagate(rk->rk_cgrp); + if (rk->rk_type == RD_KAFKA_CONSUMER && rk->rk_cgrp && rk->rk_cgrp->rkcg_group_protocol == RD_KAFKA_GROUP_PROTOCOL_CLASSIC) rd_interval_reset(&rk->rk_cgrp->rkcg_join_intvl); @@ -1208,6 +1233,7 @@ rd_kafka_parse_Metadata_admin(rd_kafka_broker_t *rkb, "(admin request)"); } +typedef RD_MAP_TYPE(const char *, const char *) map_str_str_t; /** * @brief Add all topics in current cached full metadata @@ -1218,7 +1244,7 @@ rd_kafka_parse_Metadata_admin(rd_kafka_broker_t *rkb, * an available topic will be added to this list with * the appropriate error set. * - * @returns the number of topics matched and added to \p list + * @returns the number of topics matched and added to \p tinfos * * @locks none * @locality any @@ -1228,13 +1254,16 @@ rd_kafka_metadata_topic_match(rd_kafka_t *rk, rd_list_t *tinfos, const rd_kafka_topic_partition_list_t *match, rd_kafka_topic_partition_list_t *errored) { - int ti, i; + int i; size_t cnt = 0; rd_kafka_topic_partition_list_t *unmatched; - rd_list_t cached_topics; - const char *topic; + const struct rd_kafka_metadata_cache_entry *rkmce; + map_str_str_t map; rd_kafka_rdlock(rk); + map = (map_str_str_t)RD_MAP_INITIALIZER( + rk->rk_metadata_cache.rkmc_cnt, rd_map_str_cmp, rd_map_str_hash, + NULL /* topic list element */, NULL /* topic list element */); /* To keep track of which patterns and topics in `match` that * did not match any topic (or matched an errored topic), we * create a set of all topics to match in `unmatched` and then @@ -1245,16 +1274,25 @@ rd_kafka_metadata_topic_match(rd_kafka_t *rk, /* For each topic in the cluster, scan through the match list * to find matching topic. */ - rd_list_init(&cached_topics, rk->rk_metadata_cache.rkmc_cnt, rd_free); - rd_kafka_metadata_cache_topics_to_list(rk, &cached_topics, rd_false); - RD_LIST_FOREACH(topic, &cached_topics, ti) { + TAILQ_FOREACH(rkmce, &rk->rk_metadata_cache.rkmc_expiry, rkmce_link) { const rd_kafka_metadata_topic_internal_t *mdti; - const rd_kafka_metadata_topic_t *mdt = - rd_kafka_metadata_cache_topic_get(rk, topic, &mdti, - rd_true /* valid */); - if (!mdt) + const rd_kafka_metadata_topic_t *mdt; + const char *topic = rkmce->rkmce_mtopic.topic; + rd_bool_t matched = rd_false; + + if (!RD_KAFKA_METADATA_CACHE_VALID(rkmce) || !topic || + RD_MAP_GET(&map, topic)) + /* We could have multiple cache entries + * with different topic id and same topic name + * in some cases */ continue; + RD_MAP_SET(&map, topic, topic); + + mdt = &rkmce->rkmce_mtopic; + mdti = &rkmce->rkmce_metadata_internal_topic; + + /* Ignore topics in blacklist */ if (rk->rk_conf.topic_blacklist && rd_kafka_pattern_match(rk->rk_conf.topic_blacklist, topic)) @@ -1271,6 +1309,15 @@ rd_kafka_metadata_topic_match(rd_kafka_t *rk, unmatched, match->elems[i].topic, RD_KAFKA_PARTITION_UA); + if (matched) + /* + * Just remove it from unmatched. + * Topic was already added to + * `tinfos` or `errored`. + */ + continue; + matched = rd_true; + if (mdt->err) { rd_kafka_topic_partition_list_add( errored, topic, RD_KAFKA_PARTITION_UA) @@ -1286,6 +1333,7 @@ rd_kafka_metadata_topic_match(rd_kafka_t *rk, } } rd_kafka_rdunlock(rk); + RD_MAP_DESTROY(&map); /* Any topics/patterns still in unmatched did not match any * existing topics, add them to `errored`. */ @@ -1298,7 +1346,6 @@ rd_kafka_metadata_topic_match(rd_kafka_t *rk, } rd_kafka_topic_partition_list_destroy(unmatched); - rd_list_destroy(&cached_topics); return cnt; } @@ -1383,7 +1430,8 @@ void rd_kafka_metadata_log(rd_kafka_t *rk, rd_kafka_dbg( rk, METADATA, fac, " Topic #%i/%i: %s with %i partitions%s%s", i, - md->topic_cnt, md->topics[i].topic, + md->topic_cnt, + rd_kafka_topic_name_str_safe(md->topics[i].topic), md->topics[i].partition_cnt, md->topics[i].err ? ": " : "", md->topics[i].err ? rd_kafka_err2str(md->topics[i].err) : ""); @@ -1494,6 +1542,85 @@ rd_kafka_metadata_refresh_topics(rd_kafka_t *rk, } +/** + * @brief Refresh metadata for topics identified by topic_id. + * + * For consumer paths whose canonical identity is topic_id + * (share consumer today): a by-id request lets the broker tell + * us which of our locally known topic_ids no longer correspond + * to a live topic, so stale rkts (e.g. those left behind by a + * topic delete+recreate) can be marked as non-existent and + * their broker-side share-session entries released. + * + * No by-id cache hint / dedup is performed; every call sends + * the request. Auto-creation, cgrp update, and subscription + * version are not exposed: auto-creation requires a topic + * name, and the cgrp-update hook is only consumed by the + * classic consumer protocol. + * + * @param rk used to look up usable broker if \p rkb is NULL. + * @param rkb use this broker, unless NULL then any usable broker + * from \p rk. + * @param topic_ids list of rd_kafka_Uuid_t * to query. + * @param reason reason of refresh, used in debug logs. + * + * @returns an error code; __UNKNOWN_TOPIC if topic_ids is empty; + * __TRANSPORT if no broker is available. + * + * @locality any + * @locks none + */ +rd_kafka_resp_err_t +rd_kafka_metadata_refresh_topic_ids(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + const rd_list_t *topic_ids, + const char *reason) { + rd_list_t q_topic_ids; + int destroy_rkb = 0; + + if (!rk) { + rd_assert(rkb); + rk = rkb->rkb_rk; + } + + if (rd_list_cnt(topic_ids) == 0) + return RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC; + + if (!rkb) { + if (!(rkb = rd_kafka_broker_any_usable( + rk, RD_POLL_NOWAIT, RD_DO_LOCK, 0, reason))) { + rd_kafka_dbg(rk, METADATA, "METADATA", + "Skipping metadata refresh of %d " + "topic(s) using id: %s: no usable " + "brokers", + rd_list_cnt(topic_ids), reason); + return RD_KAFKA_RESP_ERR__TRANSPORT; + } + destroy_rkb = 1; + } + + rd_list_init(&q_topic_ids, rd_list_cnt(topic_ids), + rd_list_Uuid_destroy); + rd_list_copy_to(&q_topic_ids, topic_ids, rd_list_Uuid_copy, NULL); + + rd_kafka_dbg(rk, METADATA, "METADATA", + "Requesting metadata for %d topic(s) using id: %s", + rd_list_cnt(&q_topic_ids), reason); + + rd_kafka_MetadataRequest( + rkb, NULL, &q_topic_ids, reason, rd_false /* allow_auto_create */, + rd_false /* cgrp_update */, -1 /* cgrp_subscription_version */, + rd_false /* force_racks */, NULL); + + rd_list_destroy(&q_topic_ids); + + if (destroy_rkb) + rd_kafka_broker_destroy(rkb); + + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + + /** * @brief Refresh metadata for known topics * @@ -1558,6 +1685,7 @@ rd_kafka_metadata_refresh_consumer_topics(rd_kafka_t *rk, rd_kafka_broker_t *rkb, const char *reason) { rd_list_t topics; + rd_list_t topic_ids; rd_kafka_resp_err_t err; rd_kafka_cgrp_t *rkcg; rd_bool_t allow_auto_create_topics = @@ -1580,6 +1708,23 @@ rd_kafka_metadata_refresh_consumer_topics(rd_kafka_t *rk, return rd_kafka_metadata_refresh_all(rk, rkb, reason); } + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + /* Share consumer identifies topics by id. Subscription + * names are resolved server-side via + * ShareGroupHeartbeat, so the only thing this + * periodic path needs to do is status-check the + * locally known topic ids. The broker's response + * marks stale ids (e.g. those left behind by a + * delete+recreate) as unknown, which drives cleanup + * of the corresponding rkts. */ + rd_list_init(&topic_ids, 8, rd_list_Uuid_destroy); + rd_kafka_local_topic_ids_to_list(rk, &topic_ids); + err = rd_kafka_metadata_refresh_topic_ids(rk, rkb, &topic_ids, + reason); + rd_list_destroy(&topic_ids); + return err; + } + rd_list_init(&topics, 8, rd_free); /* Add locally known topics, i.e., those that are currently @@ -2096,9 +2241,10 @@ rd_kafka_metadata_update_op(rd_kafka_t *rk, rd_kafka_metadata_internal_t *mdi) { "Topic id %s not found in cache", rd_kafka_Uuid_base64str(&topic_id)); } else { - rd_kafka_log(rk, LOG_WARNING, "METADATAUPDATE", - "Topic %s not found in cache", - topic); + rd_kafka_log( + rk, LOG_WARNING, "METADATAUPDATE", + "Topic %s not found in cache", + rd_kafka_topic_name_str_safe(topic)); } continue; } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_metadata.h b/lib/librdkafka-2.15.0/src/rdkafka_metadata.h similarity index 98% rename from lib/librdkafka-2.10.1/src/rdkafka_metadata.h rename to lib/librdkafka-2.15.0/src/rdkafka_metadata.h index 7916dcea777..343301c224e 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_metadata.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_metadata.h @@ -150,6 +150,11 @@ rd_kafka_metadata_refresh_topics(rd_kafka_t *rk, int32_t cgrp_subscription_version, const char *reason); rd_kafka_resp_err_t +rd_kafka_metadata_refresh_topic_ids(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + const rd_list_t *topic_ids, + const char *reason); +rd_kafka_resp_err_t rd_kafka_metadata_refresh_known_topics(rd_kafka_t *rk, rd_kafka_broker_t *rkb, rd_bool_t force, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_metadata_cache.c b/lib/librdkafka-2.15.0/src/rdkafka_metadata_cache.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_metadata_cache.c rename to lib/librdkafka-2.15.0/src/rdkafka_metadata_cache.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_mock.c b/lib/librdkafka-2.15.0/src/rdkafka_mock.c similarity index 80% rename from lib/librdkafka-2.10.1/src/rdkafka_mock.c rename to lib/librdkafka-2.15.0/src/rdkafka_mock.c index 3f903b27027..fee21ce1fb6 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_mock.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_mock.c @@ -51,6 +51,8 @@ rd_kafka_mock_request_new(int32_t id, int16_t api_key, int64_t timestamp_us); static void rd_kafka_mock_request_free(void *element); static void rd_kafka_mock_coord_remove(rd_kafka_mock_cluster_t *mcluster, int32_t broker_id); +static void +rd_kafka_mock_error_stack_destroy(rd_kafka_mock_error_stack_t *errstack); static rd_kafka_mock_broker_t * rd_kafka_mock_broker_find(const rd_kafka_mock_cluster_t *mcluster, @@ -185,6 +187,251 @@ rd_kafka_mock_msgset_find(const rd_kafka_mock_partition_t *mpart, return NULL; } +/** + * @brief Find or create a transaction state for the given TransactionalId. + * + * @locks mcluster->lock MUST be held. + */ +rd_kafka_mock_txn_t * +rd_kafka_mock_txn_get(rd_kafka_mock_cluster_t *mcluster, + const rd_kafkap_str_t *TransactionalId) { + rd_kafka_mock_txn_t *mtxn; + + TAILQ_FOREACH(mtxn, &mcluster->transactions, link) { + if (!rd_kafkap_str_cmp_str(TransactionalId, + mtxn->transactional_id)) + return mtxn; + } + + /* Create new */ + mtxn = rd_calloc(1, sizeof(*mtxn)); + mtxn->transactional_id = rd_strndup(TransactionalId->str, + RD_KAFKAP_STR_LEN(TransactionalId)); + mtxn->state = RD_KAFKA_MOCK_TXN_NONE; + TAILQ_INIT(&mtxn->partitions); + mtxn->partition_cnt = 0; + TAILQ_INSERT_TAIL(&mcluster->transactions, mtxn, link); + + return mtxn; +} + +/** + * @brief Find a txn partition entry, or NULL. + */ +static rd_kafka_mock_txn_partition_t * +rd_kafka_mock_txn_partition_find(rd_kafka_mock_txn_t *mtxn, + const char *topic, + int32_t partition) { + rd_kafka_mock_txn_partition_t *mtxnp; + TAILQ_FOREACH(mtxnp, &mtxn->partitions, link) { + if (mtxnp->partition == partition && + !strcmp(mtxnp->topic_name, topic)) + return mtxnp; + } + return NULL; +} + +/** + * @brief Add a partition to the transaction if not already present. + */ +void rd_kafka_mock_txn_partition_add(rd_kafka_mock_txn_t *mtxn, + const char *topic, + int32_t partition) { + rd_kafka_mock_txn_partition_t *mtxnp; + + mtxnp = rd_kafka_mock_txn_partition_find(mtxn, topic, partition); + if (mtxnp) + return; + + mtxnp = rd_calloc(1, sizeof(*mtxnp)); + mtxnp->topic_name = rd_strdup(topic); + mtxnp->partition = partition; + mtxnp->first_offset = -1; + TAILQ_INSERT_TAIL(&mtxn->partitions, mtxnp, link); + mtxn->partition_cnt++; +} + +/** + * @brief Clear all partitions from a transaction (after commit/abort). + */ +void rd_kafka_mock_txn_partitions_clear(rd_kafka_mock_txn_t *mtxn) { + rd_kafka_mock_txn_partition_t *mtxnp, *tmp; + TAILQ_FOREACH_SAFE(mtxnp, &mtxn->partitions, link, tmp) { + TAILQ_REMOVE(&mtxn->partitions, mtxnp, link); + rd_free(mtxnp->topic_name); + rd_free(mtxnp); + } + mtxn->partition_cnt = 0; +} + +/** + * @brief Destroy a transaction state object and all its partitions. + * Removes the txn from the cluster's transactions list. + */ +static void rd_kafka_mock_txn_destroy(rd_kafka_mock_cluster_t *mcluster, + rd_kafka_mock_txn_t *mtxn) { + TAILQ_REMOVE(&mcluster->transactions, mtxn, link); + rd_kafka_mock_txn_partitions_clear(mtxn); + rd_free(mtxn->transactional_id); + rd_free(mtxn); +} + +/** + * @brief Recalculate LSO for a partition. + * + * LSO = min(first_offset of all open transactions on this partition). + * If no open transactions, LSO = end_offset. + * + * @locks mcluster->lock MUST be held. + */ +void rd_kafka_mock_partition_update_lso(rd_kafka_mock_partition_t *mpart, + rd_kafka_mock_cluster_t *mcluster) { + rd_kafka_mock_txn_t *mtxn; + int64_t min_offset = mpart->end_offset; + + TAILQ_FOREACH(mtxn, &mcluster->transactions, link) { + rd_kafka_mock_txn_partition_t *mtxnp; + if (mtxn->state != RD_KAFKA_MOCK_TXN_ONGOING) + continue; + TAILQ_FOREACH(mtxnp, &mtxn->partitions, link) { + if (mtxnp->first_offset >= 0 && + mtxnp->partition == mpart->id && + !strcmp(mtxnp->topic_name, mpart->topic->name) && + mtxnp->first_offset < min_offset) + min_offset = mtxnp->first_offset; + } + } + + mpart->lso = min_offset; +} + +/** + * @brief Write a control record batch (COMMIT or ABORT) to a partition. + * + * Constructs a minimal valid MsgSet V2 with: + * - Attributes bits 4 (transactional) + 5 (control) set + * - ProducerId and ProducerEpoch from the transaction PID + * - 1 record with key = 4 bytes: schema(2) + ControlType(2) + * ControlType: 0=ABORT, 1=COMMIT + * + * The control batch occupies 1 offset in the partition log. + */ +void rd_kafka_mock_partition_write_control_batch( + rd_kafka_mock_partition_t *mpart, + rd_kafka_pid_t pid, + rd_bool_t committed) { + /* + * MsgSet V2 wire format (minimal, 1 record, no compression): + * + * 0: BaseOffset (i64) - overwritten by msgset_new + * 8: Length (i32) + * 12: PartitionLeaderEpoch (i32) - overwritten by msgset_new + * 16: MagicByte (i8) = 2 + * 17: CRC (i32) = 0 (mock doesn't validate CRC) + * 21: Attributes (i16) = 0x0030 + * 23: LastOffsetDelta (i32) = 0 + * 27: BaseTimestamp (i64) = 0 + * 35: MaxTimestamp (i64) = 0 + * 43: ProducerId (i64) + * 51: ProducerEpoch (i16) + * 53: BaseSequence (i32) = 0 + * 57: RecordCount (i32) = 1 + * + * Record (varint-encoded): + * 61: Length (varint) = 16 -> zigzag = 32 + * 62: Attributes (i8) = 0 + * 63: TimestampDelta (varint) = 0 + * 64: OffsetDelta (varint) = 0 + * 65: KeyLength (varint) = 4 -> zigzag = 8 + * 66: Key = [schema(2), ControlType(2)] + * 70: ValueLength (varint) = 6 -> zigzag = 12 + * 71: Value = 6 zero bytes (schema=0, coord_epoch=0) + * 77: HeadersCount (varint) = 0 + * Total: 78 bytes + */ + uint8_t buf[78]; + int64_t i64val; + int32_t i32val; + int16_t i16val; + rd_kafkap_bytes_t bytes; + + memset(buf, 0, sizeof(buf)); + + /* BaseOffset (placeholder) */ + i64val = htobe64(0); + memcpy(buf + 0, &i64val, 8); + + /* Length = 78 - 8(BaseOffset) - 4(Length) = 66 */ + i32val = htobe32(66); + memcpy(buf + 8, &i32val, 4); + + /* PartitionLeaderEpoch (placeholder) */ + i32val = htobe32(0); + memcpy(buf + 12, &i32val, 4); + + /* MagicByte = 2 */ + buf[16] = 2; + + /* CRC = 0 (mock doesn't validate) */ + + /* Attributes = transactional(0x10) | control(0x20) = 0x0030 */ + i16val = htobe16(0x0030); + memcpy(buf + 21, &i16val, 2); + + /* LastOffsetDelta = 0 */ + i32val = htobe32(0); + memcpy(buf + 23, &i32val, 4); + + /* BaseTimestamp = 0, MaxTimestamp = 0 (already zeroed) */ + + /* ProducerId */ + i64val = htobe64(pid.id); + memcpy(buf + 43, &i64val, 8); + + /* ProducerEpoch */ + i16val = htobe16(pid.epoch); + memcpy(buf + 51, &i16val, 2); + + /* BaseSequence = 0 (already zeroed) */ + + /* RecordCount = 1 */ + i32val = htobe32(1); + memcpy(buf + 57, &i32val, 4); + + /* Record: Length=16 (varint zigzag(16)=32) */ + buf[61] = 32; + /* Record: Attributes=0, TimestampDelta=0, OffsetDelta=0 */ + /* (already zeroed at 62, 63, 64) */ + + /* Record: KeyLength=4 (varint zigzag(4)=8) */ + buf[65] = 8; + + /* Record: Key = [schema(0x0000), ControlType] */ + buf[66] = 0; + buf[67] = 0; + buf[68] = 0; + buf[69] = committed ? 1 : 0; + + /* Record: ValueLength=6 (varint zigzag(6)=12) */ + buf[70] = 12; + + /* Record: Value = 6 zero bytes (already zeroed at 71-76) */ + + /* Record: HeadersCount=0 (already zeroed at 77) */ + + bytes.len = sizeof(buf); + bytes.data = buf; + + /* msgset_new copies the bytes, assigns offsets, and appends. */ + rd_kafka_mock_msgset_new(mpart, &bytes, 1); + + rd_kafka_dbg(mpart->topic->cluster->rk, MOCK, "MOCK", + "Wrote %s control batch to %s [%" PRId32 + "] at offset %" PRId64, + committed ? "COMMIT" : "ABORT", mpart->topic->name, + mpart->id, mpart->end_offset - 1); +} + /** * @brief Looks up or creates a new pidstate for the given partition and PID. @@ -363,6 +610,31 @@ rd_kafka_mock_partition_log_append(rd_kafka_mock_partition_t *mpart, *BaseOffset = mset->first_offset; + /* Track first offset for open transaction. */ + if (TransactionalId && RD_KAFKAP_STR_LEN(TransactionalId) > 0 && + (Attributes & RD_KAFKA_MSGSET_V2_ATTR_TRANSACTIONAL) && + !(Attributes & RD_KAFKA_MSGSET_V2_ATTR_CONTROL)) { + rd_kafka_mock_txn_t *mtxn; + rd_kafka_mock_txn_partition_t *mtxnp; + + mtx_lock(&mpart->topic->cluster->lock); + mtxn = rd_kafka_mock_txn_get(mpart->topic->cluster, + TransactionalId); + if (mtxn->state == RD_KAFKA_MOCK_TXN_ONGOING) { + mtxnp = rd_kafka_mock_txn_partition_find( + mtxn, mpart->topic->name, mpart->id); + if (mtxnp && mtxnp->first_offset == -1) + mtxnp->first_offset = mset->first_offset; + } + mtx_unlock(&mpart->topic->cluster->lock); + } + + /* Update LSO after every append — if no open transactions, + * LSO advances to end_offset. */ + mtx_lock(&mpart->topic->cluster->lock); + rd_kafka_mock_partition_update_lso(mpart, mpart->topic->cluster); + mtx_unlock(&mpart->topic->cluster->lock); + return RD_KAFKA_RESP_ERR_NO_ERROR; err_parse: @@ -436,12 +708,21 @@ static void rd_kafka_mock_partition_assign_replicas(rd_kafka_mock_partition_t *mpart, int replication_factor) { rd_kafka_mock_cluster_t *mcluster = mpart->topic->cluster; - int replica_cnt = RD_MIN(replication_factor, mcluster->broker_cnt); rd_kafka_mock_broker_t *mrkb; + int eligible_broker_cnt = 0; + int replica_cnt; int i = 0; int first_replica; int skipped = 0; + /* Count brokers eligible to be replicas: must be in metadata. */ + TAILQ_FOREACH(mrkb, &mcluster->brokers, link) { + if (mrkb->in_metadata) + eligible_broker_cnt++; + } + + replica_cnt = RD_MIN(replication_factor, eligible_broker_cnt); + if (mpart->replicas) rd_free(mpart->replicas); @@ -454,12 +735,14 @@ rd_kafka_mock_partition_assign_replicas(rd_kafka_mock_partition_t *mpart, return; } - first_replica = (mpart->id * replication_factor) % mcluster->broker_cnt; + first_replica = (mpart->id * replication_factor) % eligible_broker_cnt; /* Use a predictable, determininistic order on a per-topic basis. * * Two loops are needed for wraparound. */ TAILQ_FOREACH(mrkb, &mcluster->brokers, link) { + if (!mrkb->in_metadata) + continue; if (skipped < first_replica) { skipped++; continue; @@ -469,6 +752,8 @@ rd_kafka_mock_partition_assign_replicas(rd_kafka_mock_partition_t *mpart, mpart->replicas[i++] = mrkb; } TAILQ_FOREACH(mrkb, &mcluster->brokers, link) { + if (!mrkb->in_metadata) + continue; if (i == mpart->replica_cnt) break; mpart->replicas[i++] = mrkb; @@ -589,6 +874,8 @@ static void rd_kafka_mock_partition_destroy(rd_kafka_mock_partition_t *mpart) { rd_kafka_mock_msgset_t *mset, *tmp; rd_kafka_mock_committed_offset_t *coff, *tmpcoff; rd_kafka_mock_partition_leader_t *mpart_leader, *tmp_mpart_leader; + rd_kafka_mock_aborted_txn_t *mabort, *tmp_mabort; + rd_kafka_mock_error_stack_t *errstack, *tmp_errstack; TAILQ_FOREACH_SAFE(mset, &mpart->msgsets, link, tmp) rd_kafka_mock_msgset_destroy(mpart, mset); @@ -602,6 +889,16 @@ static void rd_kafka_mock_partition_destroy(rd_kafka_mock_partition_t *mpart) { rd_list_destroy(&mpart->pidstates); + TAILQ_FOREACH_SAFE(mabort, &mpart->aborted_txns, link, tmp_mabort) { + TAILQ_REMOVE(&mpart->aborted_txns, mabort, link); + rd_free(mabort); + } + + TAILQ_FOREACH_SAFE(errstack, &mpart->errstacks, link, tmp_errstack) { + TAILQ_REMOVE(&mpart->errstacks, errstack, link); + rd_kafka_mock_error_stack_destroy(errstack); + } + rd_free(mpart->replicas); } @@ -627,9 +924,13 @@ static void rd_kafka_mock_partition_init(rd_kafka_mock_topic_t *mtopic, TAILQ_INIT(&mpart->committed_offsets); TAILQ_INIT(&mpart->leader_responses); + TAILQ_INIT(&mpart->errstacks); rd_list_init(&mpart->pidstates, 0, rd_free); + TAILQ_INIT(&mpart->aborted_txns); + mpart->lso = 0; + rd_kafka_mock_partition_assign_replicas(mpart, replication_factor); } @@ -957,6 +1258,8 @@ static void rd_kafka_mock_connection_close(rd_kafka_mock_connection_t *mconn, reason); rd_kafka_mock_cgrps_connection_closed(mconn->broker->cluster, mconn); + rd_kafka_mock_sharegrps_node_connection_closed(mconn->broker->cluster, + mconn->broker->id); rd_kafka_timer_stop(&mconn->broker->cluster->timers, &mconn->write_tmr, rd_true); @@ -1362,8 +1665,14 @@ static void rd_kafka_mock_connection_io(rd_kafka_mock_cluster_t *mcluster, } } - if (events & (POLLERR | POLLHUP)) { - rd_kafka_mock_connection_close(mconn, "Disconnected"); + if (events & POLLERR) { + rd_kafka_mock_connection_close(mconn, + "Disconnected: " + "Error condition"); + return; + } + if (events & POLLHUP) { + rd_kafka_mock_connection_close(mconn, "Disconnected: Hang up"); return; } @@ -1484,6 +1793,8 @@ static int rd_kafka_mock_cluster_thread_main(void *arg) { rd_kafka_set_thread_name("mock"); rd_kafka_set_thread_sysname("rdk:mock"); + rd_kafka_thread_srand(mcluster->rk, + rd_true /* we're in an internal thread */); rd_kafka_interceptors_on_thread_start(mcluster->rk, RD_KAFKA_THREAD_BACKGROUND); rd_atomic32_add(&rd_kafka_thread_cnt_curr, 1); @@ -1612,6 +1923,18 @@ rd_kafka_mock_broker_decommission(rd_kafka_mock_cluster_t *mcluster, rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); } +rd_kafka_resp_err_t +rd_kafka_mock_broker_remove_from_metadata(rd_kafka_mock_cluster_t *mcluster, + int32_t broker_id) { + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_MOCK); + + rko->rko_u.mock.broker_id = broker_id; + rko->rko_u.mock.cmd = RD_KAFKA_MOCK_CMD_BROKER_REMOVE_FROM_METADATA; + + return rd_kafka_op_err_destroy( + rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); +} + rd_kafka_resp_err_t rd_kafka_mock_broker_add(rd_kafka_mock_cluster_t *mcluster, int32_t broker_id) { rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_MOCK); @@ -1745,12 +2068,13 @@ rd_kafka_mock_broker_new(rd_kafka_mock_cluster_t *mcluster, */ mrkb = rd_calloc(1, sizeof(*mrkb)); - mrkb->id = broker_id; - mrkb->cluster = mcluster; - mrkb->up = rd_true; - mrkb->listen_s = listen_s; - mrkb->sin = sin; - mrkb->port = ntohs(sin.sin_port); + mrkb->id = broker_id; + mrkb->cluster = mcluster; + mrkb->up = rd_true; + mrkb->in_metadata = rd_true; + mrkb->listen_s = listen_s; + mrkb->sin = sin; + mrkb->port = ntohs(sin.sin_port); rd_snprintf(mrkb->advertised_listener, sizeof(mrkb->advertised_listener), "%s", rd_sockaddr2str(&sin, 0)); @@ -2002,6 +2326,27 @@ rd_kafka_mock_next_request_error(rd_kafka_mock_connection_t *mconn, return err_rtt.err; } +/** + * @brief Removes and returns the next partition request error for + * request type \p ApiKey, or RD_KAFKA_RESP_ERR_NO_ERROR if + * no error was injected for this partition and ApiKey. + * + * @locality mcluster thread + */ +rd_kafka_resp_err_t +rd_kafka_mock_partition_next_request_error(rd_kafka_mock_partition_t *mpart, + int16_t ApiKey) { + rd_kafka_mock_error_stack_t *errstack; + rd_kafka_mock_error_rtt_t err_rtt; + + errstack = rd_kafka_mock_error_stack_find(&mpart->errstacks, ApiKey); + if (likely(!errstack)) + return RD_KAFKA_RESP_ERR_NO_ERROR; + + err_rtt = rd_kafka_mock_error_stack_next(errstack); + + return err_rtt.err; +} void rd_kafka_mock_clear_request_errors(rd_kafka_mock_cluster_t *mcluster, int16_t ApiKey) { @@ -2165,6 +2510,34 @@ rd_kafka_mock_topic_create(rd_kafka_mock_cluster_t *mcluster, rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); } +rd_kafka_resp_err_t +rd_kafka_mock_topic_delete(rd_kafka_mock_cluster_t *mcluster, + const char *topic) { + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_MOCK); + + rko->rko_u.mock.name = rd_strdup(topic); + rko->rko_u.mock.cmd = RD_KAFKA_MOCK_CMD_TOPIC_DELETE; + + return rd_kafka_op_err_destroy( + rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); +} + +rd_kafka_resp_err_t +rd_kafka_mock_partition_delete_records(rd_kafka_mock_cluster_t *mcluster, + const char *topic, + int32_t partition, + int64_t before_offset) { + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_MOCK); + + rko->rko_u.mock.name = rd_strdup(topic); + rko->rko_u.mock.cmd = RD_KAFKA_MOCK_CMD_PART_DELETE_RECORDS; + rko->rko_u.mock.partition = partition; + rko->rko_u.mock.lo = before_offset; + + return rd_kafka_op_err_destroy( + rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); +} + rd_kafka_resp_err_t rd_kafka_mock_partition_set_leader(rd_kafka_mock_cluster_t *mcluster, const char *topic, @@ -2232,6 +2605,34 @@ rd_kafka_mock_partition_push_leader_response(rd_kafka_mock_cluster_t *mcluster, rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); } +rd_kafka_resp_err_t +rd_kafka_mock_partition_push_request_errors(rd_kafka_mock_cluster_t *mcluster, + const char *topic, + int32_t partition, + int16_t ApiKey, + size_t cnt, + ...) { + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_MOCK); + va_list ap; + size_t i; + + rko->rko_u.mock.cmd = RD_KAFKA_MOCK_CMD_PART_PUSH_REQUEST_ERRORS; + rko->rko_u.mock.name = rd_strdup(topic); + rko->rko_u.mock.partition = partition; + rko->rko_u.mock.lo = (int64_t)ApiKey; + rko->rko_u.mock.hi = (int64_t)cnt; + rko->rko_u.mock.errs = rd_malloc(sizeof(*rko->rko_u.mock.errs) * cnt); + + va_start(ap, cnt); + for (i = 0; i < cnt; i++) + rko->rko_u.mock.errs[i] = va_arg(ap, rd_kafka_resp_err_t); + va_end(ap); + + return rd_kafka_op_err_destroy( + rd_kafka_op_req(mcluster->ops, rko, RD_POLL_INFINITE)); +} + + rd_kafka_resp_err_t rd_kafka_mock_broker_set_down(rd_kafka_mock_cluster_t *mcluster, int32_t broker_id) { @@ -2432,6 +2833,16 @@ rd_kafka_mock_broker_cmd(rd_kafka_mock_cluster_t *mcluster, mrkb->rack = NULL; break; + case RD_KAFKA_MOCK_CMD_BROKER_REMOVE_FROM_METADATA: + /* Hide the broker from Metadata responses and reassign + * partitions away from it. The broker keeps its TCP + * connections so any in-flight requests can still finish + * (or be failed by the client's metadata-driven decommission + * path with __DESTROY_BROKER). */ + mrkb->in_metadata = rd_false; + rd_kafka_mock_cluster_reassign_partitions(mcluster); + break; + case RD_KAFKA_MOCK_CMD_BROKER_DECOMMISSION: rd_kafka_mock_broker_destroy(mrkb); rd_kafka_mock_cluster_reassign_partitions(mcluster); @@ -2490,7 +2901,9 @@ rd_kafka_mock_cluster_cmd(rd_kafka_mock_cluster_t *mcluster, rd_kafka_mock_topic_t *mtopic; rd_kafka_mock_partition_t *mpart; rd_kafka_mock_broker_t *mrkb; + rd_kafka_mock_error_stack_t *errstack; size_t i; + size_t errcnt; rd_kafka_resp_err_t err; switch (rko->rko_u.mock.cmd) { @@ -2506,6 +2919,52 @@ rd_kafka_mock_cluster_cmd(rd_kafka_mock_cluster_t *mcluster, return RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION; break; + case RD_KAFKA_MOCK_CMD_TOPIC_DELETE: + mtopic = + rd_kafka_mock_topic_find(mcluster, rko->rko_u.mock.name); + if (!mtopic) + return RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + + rd_kafka_mock_topic_destroy(mtopic); + break; + + case RD_KAFKA_MOCK_CMD_PART_DELETE_RECORDS: { + rd_kafka_mock_msgset_t *mset, *tmp; + int64_t before_offset = rko->rko_u.mock.lo; + + mtopic = + rd_kafka_mock_topic_find(mcluster, rko->rko_u.mock.name); + if (!mtopic) + return RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + mpart = rd_kafka_mock_partition_find(mtopic, + rko->rko_u.mock.partition); + if (!mpart) + return RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + + if (before_offset > mpart->end_offset) + return RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE; + + /* Remove all msgsets fully before before_offset */ + TAILQ_FOREACH_SAFE(mset, &mpart->msgsets, link, tmp) { + if (mset->last_offset >= before_offset) + break; + rd_kafka_mock_msgset_destroy(mpart, mset); + } + + /* Advance start offset */ + mpart->start_offset = before_offset; + if (mpart->update_follower_start_offset) + mpart->follower_start_offset = mpart->start_offset; + + rd_kafka_dbg( + mcluster->rk, MOCK, "MOCK", + "Delete records %s [%" PRId32 "] before offset %" PRId64 + " (log now %" PRId64 "..%" PRId64 ")", + rko->rko_u.mock.name, rko->rko_u.mock.partition, + before_offset, mpart->start_offset, mpart->end_offset); + break; + } + case RD_KAFKA_MOCK_CMD_TOPIC_SET_ERROR: mtopic = rd_kafka_mock_topic_get(mcluster, rko->rko_u.mock.name, -1); @@ -2598,11 +3057,44 @@ rd_kafka_mock_cluster_cmd(rd_kafka_mock_cluster_t *mcluster, rko->rko_u.mock.leader_epoch); break; + case RD_KAFKA_MOCK_CMD_PART_PUSH_REQUEST_ERRORS: + errcnt = (size_t)rko->rko_u.mock.hi; + + mpart = rd_kafka_mock_partition_get( + mcluster, rko->rko_u.mock.name, rko->rko_u.mock.partition); + if (!mpart) + return RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + + rd_kafka_dbg(mcluster->rk, MOCK, "MOCK", + "Push %s [%" PRId32 "] %" PRIusz + " request error(s) for ApiKey %s", + rko->rko_u.mock.name, rko->rko_u.mock.partition, + errcnt, + rd_kafka_ApiKey2str((int16_t)rko->rko_u.mock.lo)); + + errstack = rd_kafka_mock_error_stack_get( + &mpart->errstacks, (int16_t)rko->rko_u.mock.lo); + + if (errstack->cnt + errcnt > errstack->size) { + errstack->size = errstack->cnt + errcnt + 4; + errstack->errs = rd_realloc( + errstack->errs, + errstack->size * sizeof(*errstack->errs)); + } + + for (i = 0; i < errcnt; i++) { + errstack->errs[errstack->cnt].err = + rko->rko_u.mock.errs[i]; + errstack->errs[errstack->cnt++].rtt = 0; + } + break; + /* Broker commands */ case RD_KAFKA_MOCK_CMD_BROKER_SET_UPDOWN: case RD_KAFKA_MOCK_CMD_BROKER_SET_RTT: case RD_KAFKA_MOCK_CMD_BROKER_SET_RACK: case RD_KAFKA_MOCK_CMD_BROKER_DECOMMISSION: + case RD_KAFKA_MOCK_CMD_BROKER_REMOVE_FROM_METADATA: return rd_kafka_mock_brokers_cmd(mcluster, rko); case RD_KAFKA_MOCK_CMD_BROKER_ADD: @@ -2700,7 +3192,9 @@ static void rd_kafka_mock_cluster_destroy0(rd_kafka_mock_cluster_t *mcluster) { rd_kafka_mock_broker_t *mrkb; rd_kafka_mock_cgrp_classic_t *mcgrp_classic; rd_kafka_mock_cgrp_consumer_t *mcgrp_consumer; + rd_kafka_mock_sharegroup_t *mshgrp; rd_kafka_mock_coord_t *mcoord; + rd_kafka_mock_txn_t *mtxn; rd_kafka_mock_error_stack_t *errstack; thrd_t dummy_rkb_thread; int ret; @@ -2718,11 +3212,17 @@ static void rd_kafka_mock_cluster_destroy0(rd_kafka_mock_cluster_t *mcluster) { while ((mcgrp_consumer = TAILQ_FIRST(&mcluster->cgrps_consumer))) rd_kafka_mock_cgrp_consumer_destroy(mcgrp_consumer); + while ((mshgrp = TAILQ_FIRST(&mcluster->sharegrps))) + rd_kafka_mock_sharegroup_destroy(mshgrp); + while ((mcoord = TAILQ_FIRST(&mcluster->coords))) rd_kafka_mock_coord_destroy(mcluster, mcoord); rd_list_destroy(&mcluster->pids); + while ((mtxn = TAILQ_FIRST(&mcluster->transactions))) + rd_kafka_mock_txn_destroy(mcluster, mtxn); + while ((errstack = TAILQ_FIRST(&mcluster->errstacks))) { TAILQ_REMOVE(&mcluster->errstacks, errstack, link); rd_kafka_mock_error_stack_destroy(errstack); @@ -2835,10 +3335,14 @@ rd_kafka_mock_cluster_t *rd_kafka_mock_cluster_new(rd_kafka_t *rk, TAILQ_INIT(&mcluster->cgrps_consumer); + rd_kafka_mock_sharegrps_init(mcluster); + TAILQ_INIT(&mcluster->coords); rd_list_init(&mcluster->pids, 16, rd_free); + TAILQ_INIT(&mcluster->transactions); + TAILQ_INIT(&mcluster->errstacks); memcpy(mcluster->api_handlers, rd_kafka_mock_api_handlers, @@ -3156,7 +3660,8 @@ static int ut_cgrp_consumer_member_next_assignment0( fixtures[i].comment); if (fixtures[i].session_timed_out) { - rd_kafka_mock_cgrp_consumer_member_leave(mcgrp, member); + rd_kafka_mock_cgrp_consumer_member_leave(mcgrp, member, + rd_false); member = rd_kafka_mock_cgrp_consumer_member_add( mcgrp, conn, &MemberId, &InstanceId, &SubscribedTopic, 1, &SubscribedTopicRegex); @@ -3644,11 +4149,366 @@ static int ut_cgrp_consumer_member_next_assignment(void) { RD_UT_PASS(); } +/** + * @brief Test that txn_get creates a new txn and reuses it. + */ +static int ut_mock_txn_get_creates_and_reuses(void) { + rd_kafka_t *rk; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_mock_txn_t *mtxn1, *mtxn2, *mtxn3; + rd_kafkap_str_t tid1 = {.str = "txn-A", .len = 5}; + rd_kafkap_str_t tid2 = {.str = "txn-B", .len = 5}; + char errstr[512]; + + RD_UT_BEGIN(); + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, errstr, sizeof(errstr)); + mcluster = rd_kafka_mock_cluster_new(rk, 1); + + mtx_lock(&mcluster->lock); + + /* First call creates a new txn */ + mtxn1 = rd_kafka_mock_txn_get(mcluster, &tid1); + RD_UT_ASSERT(mtxn1 != NULL, "txn_get returned NULL"); + RD_UT_ASSERT(mtxn1->state == RD_KAFKA_MOCK_TXN_NONE, + "new txn state should be NONE"); + RD_UT_ASSERT(!strcmp(mtxn1->transactional_id, "txn-A"), + "transactional_id mismatch"); + + /* Second call with same id returns same pointer */ + mtxn2 = rd_kafka_mock_txn_get(mcluster, &tid1); + RD_UT_ASSERT(mtxn1 == mtxn2, "same id should return same pointer"); + + /* Different id returns different pointer */ + mtxn3 = rd_kafka_mock_txn_get(mcluster, &tid2); + RD_UT_ASSERT(mtxn3 != mtxn1, + "different id should be different pointer"); + RD_UT_ASSERT(!strcmp(mtxn3->transactional_id, "txn-B"), + "transactional_id mismatch for txn-B"); + + mtx_unlock(&mcluster->lock); + + rd_kafka_mock_cluster_destroy(mcluster); + rd_kafka_destroy(rk); + + RD_UT_PASS(); +} + +/** + * @brief Test partition add, dedup, and clear. + */ +static int ut_mock_txn_partition_add_and_clear(void) { + rd_kafka_t *rk; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_mock_txn_t *mtxn; + rd_kafkap_str_t tid = {.str = "txn-parts", .len = 9}; + char errstr[512]; + + RD_UT_BEGIN(); + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, errstr, sizeof(errstr)); + mcluster = rd_kafka_mock_cluster_new(rk, 1); + + mtx_lock(&mcluster->lock); + mtxn = rd_kafka_mock_txn_get(mcluster, &tid); + + /* Add 3 different partitions */ + rd_kafka_mock_txn_partition_add(mtxn, "topic-a", 0); + rd_kafka_mock_txn_partition_add(mtxn, "topic-a", 1); + rd_kafka_mock_txn_partition_add(mtxn, "topic-b", 0); + RD_UT_ASSERT(mtxn->partition_cnt == 3, "expected 3 partitions, got %d", + mtxn->partition_cnt); + + /* Adding duplicate should not increase count */ + rd_kafka_mock_txn_partition_add(mtxn, "topic-a", 0); + RD_UT_ASSERT(mtxn->partition_cnt == 3, "expected 3 after dup, got %d", + mtxn->partition_cnt); + + /* Clear should empty the list */ + rd_kafka_mock_txn_partitions_clear(mtxn); + RD_UT_ASSERT(mtxn->partition_cnt == 0, "expected 0 after clear, got %d", + mtxn->partition_cnt); + RD_UT_ASSERT(TAILQ_EMPTY(&mtxn->partitions), + "partition list should be empty"); + + mtx_unlock(&mcluster->lock); + + rd_kafka_mock_cluster_destroy(mcluster); + rd_kafka_destroy(rk); + + RD_UT_PASS(); +} + +/** + * @brief Test that write_control_batch appends 1 offset with correct attrs. + */ +static int ut_mock_txn_control_batch_written(void) { + rd_kafka_t *rk; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_mock_topic_t *mtopic; + rd_kafka_mock_partition_t *mpart; + rd_kafka_pid_t pid = {.id = 1000, .epoch = 0}; + rd_kafka_mock_msgset_t *mset; + int16_t attrs; + int64_t offset_before; + char errstr[512]; + + RD_UT_BEGIN(); + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, errstr, sizeof(errstr)); + mcluster = rd_kafka_mock_cluster_new(rk, 1); + mtopic = rd_kafka_mock_topic_new(mcluster, "ctrl-test", 1, 1); + mpart = &mtopic->partitions[0]; + + offset_before = mpart->end_offset; + + /* Write a COMMIT control batch */ + rd_kafka_mock_partition_write_control_batch(mpart, pid, rd_true); + + /* end_offset should have increased by 1 */ + RD_UT_ASSERT(mpart->end_offset == offset_before + 1, + "expected end_offset=%" PRId64 ", got %" PRId64, + offset_before + 1, mpart->end_offset); + + /* Check the last msgset has transactional+control attributes */ + mset = TAILQ_LAST(&mpart->msgsets, rd_kafka_mock_msgset_tailq_s); + RD_UT_ASSERT(mset != NULL, "no msgset found"); + + memcpy(&attrs, + (const char *)mset->bytes.data + + RD_KAFKAP_MSGSET_V2_OF_Attributes, + sizeof(attrs)); + attrs = be16toh(attrs); + RD_UT_ASSERT((attrs & 0x0030) == 0x0030, + "expected attrs bits 4+5 set (0x0030), got 0x%04x", attrs); + + /* Write an ABORT control batch */ + offset_before = mpart->end_offset; + rd_kafka_mock_partition_write_control_batch(mpart, pid, rd_false); + RD_UT_ASSERT(mpart->end_offset == offset_before + 1, + "ABORT: expected end_offset=%" PRId64 ", got %" PRId64, + offset_before + 1, mpart->end_offset); + + rd_kafka_mock_cluster_destroy(mcluster); + rd_kafka_destroy(rk); + + RD_UT_PASS(); +} + +/** + * @brief Test LSO calculation with open and closed transactions. + */ +static int ut_mock_txn_lso_calculation(void) { + rd_kafka_t *rk; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_mock_topic_t *mtopic; + rd_kafka_mock_partition_t *mpart; + rd_kafka_mock_txn_t *mtxn; + rd_kafkap_str_t tid = {.str = "txn-lso", .len = 7}; + char errstr[512]; + + RD_UT_BEGIN(); + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, errstr, sizeof(errstr)); + mcluster = rd_kafka_mock_cluster_new(rk, 1); + mtopic = rd_kafka_mock_topic_new(mcluster, "lso-test", 1, 1); + mpart = &mtopic->partitions[0]; + + /* No open txns → LSO = end_offset */ + mtx_lock(&mcluster->lock); + rd_kafka_mock_partition_update_lso(mpart, mcluster); + RD_UT_ASSERT(mpart->lso == mpart->end_offset, + "no txns: expected lso=%" PRId64 ", got %" PRId64, + mpart->end_offset, mpart->lso); + + /* Simulate some records in the log so end_offset > 5 */ + mpart->end_offset = 10; + + /* Create an open txn with first_offset=5 on this partition */ + mtxn = rd_kafka_mock_txn_get(mcluster, &tid); + mtxn->state = RD_KAFKA_MOCK_TXN_ONGOING; + rd_kafka_mock_txn_partition_add(mtxn, "lso-test", 0); + { + rd_kafka_mock_txn_partition_t *mtxnp; + TAILQ_FOREACH(mtxnp, &mtxn->partitions, link) { + if (!strcmp(mtxnp->topic_name, "lso-test") && + mtxnp->partition == 0) + mtxnp->first_offset = 5; + } + } + + rd_kafka_mock_partition_update_lso(mpart, mcluster); + RD_UT_ASSERT(mpart->lso == 5, + "open txn at 5: expected lso=5, got %" PRId64, mpart->lso); + + /* Close the txn → LSO should go back to end_offset */ + mtxn->state = RD_KAFKA_MOCK_TXN_NONE; + rd_kafka_mock_partition_update_lso(mpart, mcluster); + RD_UT_ASSERT(mpart->lso == mpart->end_offset, + "closed txn: expected lso=%" PRId64 ", got %" PRId64, + mpart->end_offset, mpart->lso); + + mtx_unlock(&mcluster->lock); + + rd_kafka_mock_cluster_destroy(mcluster); + rd_kafka_destroy(rk); + + RD_UT_PASS(); +} + +/** + * @brief Test that aborted transaction range is recorded on partition. + */ +static int ut_mock_txn_aborted_range_recorded(void) { + rd_kafka_t *rk; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_mock_topic_t *mtopic; + rd_kafka_mock_partition_t *mpart; + rd_kafka_mock_aborted_txn_t *mabort; + rd_kafka_pid_t pid = {.id = 2000, .epoch = 0}; + int64_t last_data_offset; + char errstr[512]; + + RD_UT_BEGIN(); + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, errstr, sizeof(errstr)); + mcluster = rd_kafka_mock_cluster_new(rk, 1); + mtopic = rd_kafka_mock_topic_new(mcluster, "abort-test", 1, 1); + mpart = &mtopic->partitions[0]; + + /* Simulate: data was produced at offsets 0,1,2 (end_offset=3) */ + /* We fake this by just setting end_offset directly */ + mpart->end_offset = 3; + + last_data_offset = mpart->end_offset - 1; /* 2 */ + + /* Write ABORT control batch → end_offset becomes 4 */ + rd_kafka_mock_partition_write_control_batch(mpart, pid, rd_false); + + /* Record the aborted range */ + mabort = rd_calloc(1, sizeof(*mabort)); + mabort->pid_id = pid.id; + mabort->first_offset = 0; + mabort->last_offset = last_data_offset; + TAILQ_INSERT_TAIL(&mpart->aborted_txns, mabort, link); + + /* Verify the aborted range */ + mabort = TAILQ_FIRST(&mpart->aborted_txns); + RD_UT_ASSERT(mabort != NULL, "expected 1 aborted txn entry"); + RD_UT_ASSERT(mabort->pid_id == 2000, "expected pid=2000, got %" PRId64, + mabort->pid_id); + RD_UT_ASSERT(mabort->first_offset == 0, + "expected first_offset=0, got %" PRId64, + mabort->first_offset); + RD_UT_ASSERT(mabort->last_offset == 2, + "expected last_offset=2, got %" PRId64, + mabort->last_offset); + + /* Control batch should have bumped end_offset to 4 */ + RD_UT_ASSERT(mpart->end_offset == 4, + "expected end_offset=4, got %" PRId64, mpart->end_offset); + + rd_kafka_mock_cluster_destroy(mcluster); + rd_kafka_destroy(rk); + + RD_UT_PASS(); +} + +/** + * @brief Test aborted offset range checking. + */ +static int ut_mock_txn_offset_is_aborted(void) { + rd_kafka_t *rk; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_mock_topic_t *mtopic; + rd_kafka_mock_partition_t *mpart; + rd_kafka_mock_aborted_txn_t *mabort; + char errstr[512]; + + RD_UT_BEGIN(); + + rk = rd_kafka_new(RD_KAFKA_CONSUMER, NULL, errstr, sizeof(errstr)); + mcluster = rd_kafka_mock_cluster_new(rk, 1); + mtopic = rd_kafka_mock_topic_new(mcluster, "isabort-test", 1, 1); + mpart = &mtopic->partitions[0]; + + /* Add aborted range: pid=1000, offsets 5-10 */ + mabort = rd_calloc(1, sizeof(*mabort)); + mabort->pid_id = 1000; + mabort->first_offset = 5; + mabort->last_offset = 10; + TAILQ_INSERT_TAIL(&mpart->aborted_txns, mabort, link); + + /* Check: offset in range + matching pid → aborted */ + { + rd_kafka_mock_aborted_txn_t *a; + rd_bool_t found; + + /* offset=5, pid=1000 → should match */ + found = rd_false; + TAILQ_FOREACH(a, &mpart->aborted_txns, link) { + if (a->pid_id == 1000 && 5 >= a->first_offset && + 5 <= a->last_offset) { + found = rd_true; + break; + } + } + RD_UT_ASSERT(found, "offset 5 pid 1000 should be aborted"); + + /* offset=7, pid=1000 → should match */ + found = rd_false; + TAILQ_FOREACH(a, &mpart->aborted_txns, link) { + if (a->pid_id == 1000 && 7 >= a->first_offset && + 7 <= a->last_offset) { + found = rd_true; + break; + } + } + RD_UT_ASSERT(found, "offset 7 pid 1000 should be aborted"); + + /* offset=11, pid=1000 → should NOT match (out of range) */ + found = rd_false; + TAILQ_FOREACH(a, &mpart->aborted_txns, link) { + if (a->pid_id == 1000 && 11 >= a->first_offset && + 11 <= a->last_offset) { + found = rd_true; + break; + } + } + RD_UT_ASSERT(!found, + "offset 11 pid 1000 should NOT be aborted"); + + /* offset=7, pid=2000 → should NOT match (wrong pid) */ + found = rd_false; + TAILQ_FOREACH(a, &mpart->aborted_txns, link) { + if (a->pid_id == 2000 && 7 >= a->first_offset && + 7 <= a->last_offset) { + found = rd_true; + break; + } + } + RD_UT_ASSERT(!found, "offset 7 pid 2000 should NOT be aborted"); + } + + rd_kafka_mock_cluster_destroy(mcluster); + rd_kafka_destroy(rk); + + RD_UT_PASS(); +} + /** * @brief Mock cluster unit tests */ int unittest_mock_cluster(void) { int fails = 0; fails += ut_cgrp_consumer_member_next_assignment(); + + fails += ut_mock_txn_get_creates_and_reuses(); + fails += ut_mock_txn_partition_add_and_clear(); + fails += ut_mock_txn_control_batch_written(); + fails += ut_mock_txn_lso_calculation(); + fails += ut_mock_txn_aborted_range_recorded(); + fails += ut_mock_txn_offset_is_aborted(); + return fails; } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_mock.h b/lib/librdkafka-2.15.0/src/rdkafka_mock.h similarity index 67% rename from lib/librdkafka-2.10.1/src/rdkafka_mock.h rename to lib/librdkafka-2.15.0/src/rdkafka_mock.h index 0b81b312ef8..2ad1bb5c401 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_mock.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_mock.h @@ -221,6 +221,40 @@ rd_kafka_mock_broker_error_stack_cnt(rd_kafka_mock_cluster_t *mcluster, int16_t ApiKey, size_t *cntp); +/** + * @brief Push \p cnt errors in the \p ... va-arg list onto the partition's + * error stack for the given \p ApiKey. + * + * \p ApiKey is the Kafka protocol request type, e.g., ProduceRequest (0). + * + * The injected errors will be returned as the partition-level ErrorCode, + * one per request, in the order they were pushed, for requests of type + * \p ApiKey referencing this partition. Other partitions in the same + * request are not affected. + * + * @remark Cluster errors (rd_kafka_mock_push_request_errors()) and broker + * errors (rd_kafka_mock_broker_push_request_error_rtts()) take + * precedence over partition errors. + * + * @remark The error codes are not validated against the set of errors + * the Apache Kafka protocol permits for the given \p ApiKey and + * response field, that is the test's responsibility. + * + * @remark If \p topic does not exist in the mock cluster it is + * auto-created, same as for the other mock partition APIs. + * + * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, or + * RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART if \p topic exists + * and \p partition is out of range. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_mock_partition_push_request_errors(rd_kafka_mock_cluster_t *mcluster, + const char *topic, + int32_t partition, + int16_t ApiKey, + size_t cnt, + ...); + /** * @brief Set the topic error to return in protocol requests. @@ -249,6 +283,47 @@ rd_kafka_mock_topic_create(rd_kafka_mock_cluster_t *mcluster, int replication_factor); +/** + * @brief Deletes a topic and all its partitions. + * + * Subsequent requests referencing the topic will receive + * UNKNOWN_TOPIC_OR_PARTITION. + * + * @param mcluster The mock cluster instance. + * @param topic The topic to delete. + * + * @return RD_KAFKA_RESP_ERR_NO_ERROR on success + * RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART if the topic does not + * exist. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_mock_topic_delete(rd_kafka_mock_cluster_t *mcluster, + const char *topic); + + +/** + * @brief Delete records before \p before_offset in \p topic [\p partition]. + * + * Advances the partition's start offset (log start offset) to + * \p before_offset and removes all message sets fully before that offset. + * + * @param mcluster The mock cluster. + * @param topic Topic name. + * @param partition Partition id. + * @param before_offset Delete all records before this offset. + * The new start offset will be set to this value. + * + * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, + * RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART if partition doesn't exist, + * RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE if before_offset > end_offset. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_mock_partition_delete_records(rd_kafka_mock_cluster_t *mcluster, + const char *topic, + int32_t partition, + int64_t before_offset); + + /** * @brief Sets the partition leader. * @@ -387,6 +462,31 @@ RD_EXPORT rd_kafka_resp_err_t rd_kafka_mock_broker_decommission(rd_kafka_mock_cluster_t *cluster, int32_t broker_id); +/** + * @brief Hide the broker from Metadata responses without dropping its + * existing TCP connections or its listen socket. + * + * Unlike rd_kafka_mock_broker_decommission(), the broker's connections + * stay alive so any in-flight requests can still complete on the + * broker side. The broker is removed from cluster metadata responses + * and partitions are reassigned away from it. This lets a client's + * metadata-driven decommission path observe the broker's removal + * and fail in-flight requests with __DESTROY_BROKER, instead of + * the connection-drop path failing them with __TRANSPORT. + * + * The broker remains in the cluster's broker list; its struct is + * not freed until the cluster is destroyed (or the broker is added + * back via the appropriate API in the future). + * + * @param mcluster The mock cluster. + * @param broker_id The broker to hide from metadata. + * + * @returns Error value or 0 if no error occurred. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_mock_broker_remove_from_metadata(rd_kafka_mock_cluster_t *mcluster, + int32_t broker_id); + /** * @brief Add a new broker to the cluster. * Cluster partition will be reassigned to use the new broker @@ -601,6 +701,163 @@ RD_EXPORT void rd_kafka_mock_set_group_consumer_heartbeat_interval_ms( rd_kafka_mock_cluster_t *mcluster, int group_consumer_heartbeat_interval_ms); +/** + * @brief Set the sharegroup session timeout in milliseconds. + * + * @param mcluster Mock cluster instance. + * @param session_timeout_ms Session timeout in milliseconds. + */ +RD_EXPORT void +rd_kafka_mock_sharegroup_set_session_timeout(rd_kafka_mock_cluster_t *mcluster, + int session_timeout_ms); + +/** + * @brief Set the sharegroup heartbeat interval in milliseconds. + * + * @param mcluster Mock cluster instance. + * @param heartbeat_interval_ms Heartbeat interval in milliseconds. + */ +RD_EXPORT void rd_kafka_mock_sharegroup_set_heartbeat_interval( + rd_kafka_mock_cluster_t *mcluster, + int heartbeat_interval_ms); + +/** + * @brief Set the maximum number of times a record can be acquired + * before it is automatically archived (dead-lettered). + * + * Default is 5. Set to 0 for unlimited deliveries. + * + * @param mcluster Mock cluster instance. + * @param max_attempts Maximum delivery attempts per record. + */ +RD_EXPORT void rd_kafka_mock_sharegroup_set_max_delivery_attempts( + rd_kafka_mock_cluster_t *mcluster, + int max_attempts); + +/** + * @brief Set the per-record lock duration in milliseconds. + * + * When a record is acquired, its lock expires after this duration. + * Default is 0, which falls back to the session timeout value. + * + * @param mcluster Mock cluster instance. + * @param lock_duration_ms Lock duration in milliseconds. + */ +RD_EXPORT void rd_kafka_mock_sharegroup_set_record_lock_duration( + rd_kafka_mock_cluster_t *mcluster, + int lock_duration_ms); + +/** + * @brief Set the share group isolation level for transactions. + * + * Controls how the share group handles transactional records: + * 0 = read_uncommitted (default) — reads up to HWM, no filtering + * 1 = read_committed — reads up to LSO, filters aborted transaction data + * + * @param mcluster Mock cluster instance. + * @param level Isolation level (0 or 1). + */ +RD_EXPORT void +rd_kafka_mock_sharegroup_set_isolation_level(rd_kafka_mock_cluster_t *mcluster, + int level); + +/** + * @brief Set the maximum number of members allowed in a share group. + * + * New members attempting to join via ShareGroupHeartbeat when the group + * is at capacity will receive GROUP_MAX_SIZE_REACHED. + * + * Default is 0 (unlimited). + * + * @param mcluster Mock cluster instance. + * @param max_size Maximum members allowed. 0 = unlimited. + */ +RD_EXPORT void +rd_kafka_mock_sharegroup_set_max_size(rd_kafka_mock_cluster_t *mcluster, + int max_size); + +/** + * @brief Set the maximum number of fetch sessions allowed per broker. + * + * New sessions attempted via ShareFetch with epoch 0 when the broker + * is at capacity will receive SHARE_SESSION_LIMIT_REACHED. + * + * Default is 2000 (group.share.max.share.sessions). + * + * @param mcluster Mock cluster instance. + * @param max_fetch_sessions Maximum fetch sessions per broker. 0 = unlimited. + */ +RD_EXPORT void rd_kafka_mock_sharegroup_set_max_fetch_sessions( + rd_kafka_mock_cluster_t *mcluster, + int max_fetch_sessions); + +/** + * @brief Set the maximum number of in-flight record locks per share-partition. + * + * Once the limit is reached, no more records are acquired until existing + * locks are released (via ack, release, reject, or lock expiry). + * + * Default is 2000 (group.share.partition.max.record.locks). + * + * @param mcluster Mock cluster instance. + * @param max_record_locks Maximum in-flight records per partition. 0 = + * unlimited. + */ +RD_EXPORT void +rd_kafka_mock_sharegroup_set_max_record_locks(rd_kafka_mock_cluster_t *mcluster, + int max_record_locks); + +/** + * @brief Set the auto offset reset policy for share groups. + * + * Controls where SPSO is initialized when a share-partition is first + * consumed. + * + * Default is 0 ("latest"). + * + * @param mcluster Mock cluster instance. + * @param auto_offset_reset 0 = latest, 1 = earliest. + */ +RD_EXPORT void rd_kafka_mock_sharegroup_set_auto_offset_reset( + rd_kafka_mock_cluster_t *mcluster, + int auto_offset_reset); + +/** + * @brief Set a manual target assignment for a sharegroup. + * + * This allows tests to override the automatic partition assignment + * and manually specify which partitions each member should get. + * + * @param mcluster Mock cluster instance. + * @param group_id The sharegroup ID. + * @param member_ids Array of member IDs (strings). + * @param assignments Array of partition lists (one per member). + * @param member_cnt Number of members (length of both arrays). + */ +RD_EXPORT void rd_kafka_mock_sharegroup_target_assignment( + rd_kafka_mock_cluster_t *mcluster, + const char *group_id, + const char **member_ids, + rd_kafka_topic_partition_list_t **assignments, + size_t member_cnt); + +/** + * @brief Retrieve the member IDs from a sharegroup. + * + * @param mcluster Mock cluster instance. + * @param group_id The sharegroup ID. + * @param member_ids_out Output array of member IDs. Caller must free each + * string with rd_free() and the array itself. + * @param member_cnt_out Output count of members. + * + * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, + * RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND if sharegroup not found. + */ +RD_EXPORT rd_kafka_resp_err_t +rd_kafka_mock_sharegroup_get_member_ids(rd_kafka_mock_cluster_t *mcluster, + const char *group_id, + char ***member_ids_out, + size_t *member_cnt_out); /**@}*/ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_mock_cgrp.c b/lib/librdkafka-2.15.0/src/rdkafka_mock_cgrp.c similarity index 97% rename from lib/librdkafka-2.10.1/src/rdkafka_mock_cgrp.c rename to lib/librdkafka-2.15.0/src/rdkafka_mock_cgrp.c index f5998b1a4d7..28978a56ec6 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_mock_cgrp.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_mock_cgrp.c @@ -1486,6 +1486,15 @@ static rd_bool_t rd_kafka_mock_cgrp_consumer_member_subscribed_topic_names_set( return changed; } +static void rd_kafka_mock_cgrp_consumer_member_topic_id_set( + rd_kafka_mock_cgrp_consumer_member_t *member, + const rd_kafkap_str_t *MemberId) { + /* KIP 1082: MemberId is generated by the client */ + rd_assert(RD_KAFKAP_STR_LEN(MemberId) > 0); + RD_IF_FREE(member->id, rd_free); + member->id = RD_KAFKAP_STR_DUP(MemberId); +} + /** * @brief Adds a member to consumer group \p mcgrp. If member with same * \p MemberId is already present, only updates the connection and @@ -1521,17 +1530,24 @@ rd_kafka_mock_cgrp_consumer_member_t *rd_kafka_mock_cgrp_consumer_member_add( if (!member) { member = rd_kafka_mock_cgrp_consumer_member_find_by_instance_id( mcgrp, InstanceId); - if (member && RD_KAFKAP_STR_LEN(MemberId) > 0 && - rd_kafkap_str_cmp_str(MemberId, member->id) != 0) { - /* Either member is a new instance and is rejoining - * with same InstanceId, so MemberId is NULL, - * or it's rejoining after unsubscribing, - * then it must have the same MemberId as before, - * as it lasts for member lifetime. - * It both don't hold, we cannot add the member - * to the group. */ - return NULL; + + if (member) { + if (!member->left_static_membership) { + /* Old member still active, + * fence this one */ + return NULL; + } + + if (rd_kafkap_str_cmp_str(MemberId, member->id) != 0) { + /* Member is a new instance and is rejoining + * with a new MemberId. */ + rd_kafka_mock_cgrp_consumer_member_topic_id_set( + member, MemberId); + } + member->left_static_membership = rd_false; } + } else { + member->left_static_membership = rd_false; } if (!member) { @@ -1548,7 +1564,8 @@ rd_kafka_mock_cgrp_consumer_member_t *rd_kafka_mock_cgrp_consumer_member_add( member = rd_calloc(1, sizeof(*member)); member->mcgrp = mcgrp; - member->id = RD_KAFKAP_STR_DUP(MemberId); + rd_kafka_mock_cgrp_consumer_member_topic_id_set(member, + MemberId); if (!RD_KAFKAP_STR_IS_NULL(InstanceId)) member->instance_id = RD_KAFKAP_STR_DUP(InstanceId); @@ -1617,28 +1634,39 @@ static void rd_kafka_mock_cgrp_consumer_member_destroy( rd_free(member); } +static void rd_kafka_mock_cgrp_consumer_member_leave_static( + rd_kafka_mock_cgrp_consumer_member_t *member) { + member->left_static_membership = rd_true; + rd_kafka_mock_cgrp_consumer_member_returned_assignment_set(member, + NULL); +} + /** * @brief Called when a member must leave a consumer group. * * @param mcgrp Consumer group to leave. * @param member Member that leaves. + * @param leave_static If true, the member is leaving with static group + * membership. * * @locks mcluster->lock MUST be held. */ void rd_kafka_mock_cgrp_consumer_member_leave( rd_kafka_mock_cgrp_consumer_t *mcgrp, - rd_kafka_mock_cgrp_consumer_member_t *member) { + rd_kafka_mock_cgrp_consumer_member_t *member, + rd_bool_t leave_static) { rd_bool_t is_static = member->instance_id != NULL; rd_kafka_dbg(mcgrp->cluster->rk, MOCK, "MOCK", - "Member %s is leaving group %s, is static: %s", member->id, - mcgrp->id, RD_STR_ToF(is_static)); - if (!is_static) + "Member %s is leaving group %s, is static: %s, " + "static leave: %s", + member->id, mcgrp->id, RD_STR_ToF(is_static), + RD_STR_ToF(leave_static)); + if (!is_static || !leave_static) rd_kafka_mock_cgrp_consumer_member_destroy(mcgrp, member); else - rd_kafka_mock_cgrp_consumer_member_returned_assignment_set( - member, NULL); + rd_kafka_mock_cgrp_consumer_member_leave_static(member); } /** @@ -1845,4 +1873,5 @@ void rd_kafka_mock_cgrps_connection_closed(rd_kafka_mock_cluster_t *mcluster, rd_kafka_mock_connection_t *mconn) { rd_kafka_mock_cgrps_classic_connection_closed(mcluster, mconn); rd_kafka_mock_cgrps_consumer_connection_closed(mcluster, mconn); + rd_kafka_mock_sharegrps_connection_closed(mcluster, mconn); } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_mock_handlers.c b/lib/librdkafka-2.15.0/src/rdkafka_mock_handlers.c similarity index 54% rename from lib/librdkafka-2.10.1/src/rdkafka_mock_handlers.c rename to lib/librdkafka-2.15.0/src/rdkafka_mock_handlers.c index 4566018da11..f75b66c7c95 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_mock_handlers.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_mock_handlers.c @@ -172,6 +172,10 @@ static int rd_kafka_mock_handle_Produce(rd_kafka_mock_connection_t *mconn, else if (mpart->leader != mconn->broker) err = RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION; + else + err = + rd_kafka_mock_partition_next_request_error( + mpart, rkbuf->rkbuf_reqhdr.ApiKey); /* Append to partition log */ if (!err) @@ -502,6 +506,11 @@ static int rd_kafka_mock_handle_Fetch(rd_kafka_mock_connection_t *mconn, rd_kafka_mock_partition_leader_epoch_check( mpart, CurrentLeaderEpoch); + if (!err && mpart) + err = + rd_kafka_mock_partition_next_request_error( + mpart, rkbuf->rkbuf_reqhdr.ApiKey); + /* Find MessageSet for FetchOffset */ if (!err && FetchOffset != mpart->end_offset) { /* Kafka currently only returns @@ -1300,6 +1309,15 @@ static int rd_kafka_mock_handle_Metadata(rd_kafka_mock_connection_t *mconn, size_t of_Brokers_cnt; int32_t response_Brokers_cnt = 0; + /* Consume the next pushed err+rtt for ApiKey=Metadata so an + * injected RTT (or __TRANSPORT close) on this connection takes + * effect. The returned err code is not propagated into the + * response body (Metadata's error reporting is per-topic and + * per-partition; tests that need to inject a topic-level + * MetadataResponse error should use the dedicated mock + * topic/partition error APIs). */ + rd_kafka_mock_next_request_error(mconn, resp); + if (rkbuf->rkbuf_reqhdr.ApiVersion >= 3) { /* Response: ThrottleTime */ rd_kafka_buf_write_i32(resp, 0); @@ -1309,7 +1327,7 @@ static int rd_kafka_mock_handle_Metadata(rd_kafka_mock_connection_t *mconn, of_Brokers_cnt = rd_kafka_buf_write_arraycnt_pos(resp); TAILQ_FOREACH(mrkb, &mcluster->brokers, link) { - if (!mrkb->up) + if (!mrkb->up || !mrkb->in_metadata) continue; /* Response: Brokers.Nodeid */ rd_kafka_buf_write_i32(resp, mrkb->id); @@ -2143,6 +2161,7 @@ rd_kafka_mock_handle_AddPartitionsToTxn(rd_kafka_mock_connection_t *mconn, rd_kafka_resp_err_t all_err; rd_kafkap_str_t TransactionalId; rd_kafka_pid_t pid; + rd_kafka_mock_txn_t *mtxn = NULL; int32_t TopicsCnt; /* Response: ThrottleTimeMs */ @@ -2172,6 +2191,15 @@ rd_kafka_mock_handle_AddPartitionsToTxn(rd_kafka_mock_connection_t *mconn, all_err = rd_kafka_mock_pid_check(mcluster, &TransactionalId, pid); + /* Track transaction state. */ + if (!all_err) { + mtx_lock(&mcluster->lock); + mtxn = rd_kafka_mock_txn_get(mcluster, &TransactionalId); + mtxn->pid = pid; + mtxn->state = RD_KAFKA_MOCK_TXN_ONGOING; + mtx_unlock(&mcluster->lock); + } + while (TopicsCnt-- > 0) { rd_kafkap_str_t Topic; int32_t PartsCnt; @@ -2204,6 +2232,16 @@ rd_kafka_mock_handle_AddPartitionsToTxn(rd_kafka_mock_connection_t *mconn, else if (mtopic && mtopic->err) err = mtopic->err; + /* Track partition in transaction */ + if (!err && mtxn) { + char *topic_name; + RD_KAFKAP_STR_DUPA(&topic_name, &Topic); + mtx_lock(&mcluster->lock); + rd_kafka_mock_txn_partition_add( + mtxn, topic_name, Partition); + mtx_unlock(&mcluster->lock); + } + /* Response: ErrorCode */ rd_kafka_buf_write_i16(resp, err); } @@ -2440,6 +2478,86 @@ static int rd_kafka_mock_handle_EndTxn(rd_kafka_mock_connection_t *mconn, if (!err) err = rd_kafka_mock_pid_check(mcluster, &TransactionalId, pid); + /* Commit/abort the transaction. */ + if (!err) { + rd_kafka_mock_txn_t *mtxn; + rd_kafka_mock_txn_partition_t *mtxnp; + + mtx_lock(&mcluster->lock); + mtxn = rd_kafka_mock_txn_get(mcluster, &TransactionalId); + + if (mtxn->state != RD_KAFKA_MOCK_TXN_ONGOING) { + /* No active transaction — could be an idempotent + * retry after the txn was already completed. */ + mtx_unlock(&mcluster->lock); + goto write_response; + } + + /* For each partition in the transaction, write a control + * batch and (for abort) record the aborted range. */ + TAILQ_FOREACH(mtxnp, &mtxn->partitions, link) { + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find(mcluster, + mtxnp->topic_name); + rd_kafka_mock_partition_t *mpart; + int64_t last_data_offset; + + if (!mtopic) + continue; + mpart = rd_kafka_mock_partition_find(mtopic, + mtxnp->partition); + if (!mpart) + continue; + + /* Skip partitions where no data was produced */ + if (mtxnp->first_offset < 0) + continue; + + /* Offset just before the control batch */ + last_data_offset = mpart->end_offset - 1; + + /* Write COMMIT or ABORT control record */ + rd_kafka_mock_partition_write_control_batch(mpart, pid, + committed); + + /* For abort: record the aborted range */ + if (!committed) { + rd_kafka_mock_aborted_txn_t *mabort = + rd_calloc(1, sizeof(*mabort)); + mabort->pid_id = pid.id; + mabort->first_offset = mtxnp->first_offset; + mabort->last_offset = last_data_offset; + TAILQ_INSERT_TAIL(&mpart->aborted_txns, mabort, + link); + } + } + + /* Clear transaction state BEFORE recalculating LSO, + * so update_lso no longer sees this txn as open. */ + mtxn->state = RD_KAFKA_MOCK_TXN_NONE; + + /* Recalculate LSO for all affected partitions now that + * the txn is no longer ONGOING. */ + TAILQ_FOREACH(mtxnp, &mtxn->partitions, link) { + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find(mcluster, + mtxnp->topic_name); + rd_kafka_mock_partition_t *mpart; + if (!mtopic) + continue; + mpart = rd_kafka_mock_partition_find(mtopic, + mtxnp->partition); + if (!mpart) + continue; + rd_kafka_mock_partition_update_lso(mpart, mcluster); + } + + rd_kafka_mock_txn_partitions_clear(mtxn); + + mtx_unlock(&mcluster->lock); + } + +write_response: /* ErrorCode */ rd_kafka_buf_write_i16(resp, err); @@ -2696,6 +2814,7 @@ static int rd_kafka_mock_handle_PushTelemetry(rd_kafka_mock_connection_t *mconn, rd_kafka_Uuid_t ClientInstanceId; int32_t SubscriptionId; rd_bool_t terminating; + int8_t CompressionType; rd_kafka_compression_t compression_type = RD_KAFKA_COMPRESSION_NONE; rd_kafkap_bytes_t metrics; rd_kafka_resp_err_t err; @@ -2703,7 +2822,8 @@ static int rd_kafka_mock_handle_PushTelemetry(rd_kafka_mock_connection_t *mconn, rd_kafka_buf_read_uuid(rkbuf, &ClientInstanceId); rd_kafka_buf_read_i32(rkbuf, &SubscriptionId); rd_kafka_buf_read_bool(rkbuf, &terminating); - rd_kafka_buf_read_i8(rkbuf, &compression_type); + rd_kafka_buf_read_i8(rkbuf, &CompressionType); + compression_type = CompressionType; rd_kafka_buf_read_kbytes(rkbuf, &metrics); void *uncompressed_payload = NULL; @@ -2875,6 +2995,16 @@ rd_kafka_mock_handle_ConsumerGroupHeartbeat(rd_kafka_mock_connection_t *mconn, err = RD_KAFKA_RESP_ERR_NOT_COORDINATOR; } + /* A member (re-)joining (MemberEpoch 0) must provide a + * RebalanceTimeoutMs and a subscription (topic names or regex), + * otherwise the coordinator rejects the request. This mirrors the + * broker-side validation and catches malformed (re-)join heartbeats. */ + if (!err && MemberEpoch == 0 && + (RebalanceTimeoutMs == -1 || + (!SubscribedTopicNames && + RD_KAFKAP_STR_IS_NULL(&SubscribedTopicRegex)))) + err = RD_KAFKA_RESP_ERR_INVALID_REQUEST; + if (!err) { mtx_lock(&mcluster->lock); mcgrp = rd_kafka_mock_cgrp_consumer_get(mcluster, &GroupId); @@ -2896,7 +3026,7 @@ rd_kafka_mock_handle_ConsumerGroupHeartbeat(rd_kafka_mock_connection_t *mconn, } } else { rd_kafka_mock_cgrp_consumer_member_leave( - mcgrp, member); + mcgrp, member, MemberEpoch == -2); member = NULL; } } else { @@ -2991,6 +3121,2015 @@ rd_kafka_mock_handle_ConsumerGroupHeartbeat(rd_kafka_mock_connection_t *mconn, return -1; } +/** + * @brief Helper to write assignment TopicPartitions to ShareGroupHeartbeat + * response. + */ +static void rd_kafka_mock_handle_ShareGroupHeartbeat_write_TopicPartitions( + rd_kafka_buf_t *resp, + rd_kafka_topic_partition_list_t *assignment) { + const rd_kafka_topic_partition_field_t fields[] = { + RD_KAFKA_TOPIC_PARTITION_FIELD_PARTITION, + RD_KAFKA_TOPIC_PARTITION_FIELD_END}; + + rd_kafka_topic_partition_list_sort_by_topic_id(assignment); + rd_kafka_buf_write_topic_partitions( + resp, assignment, rd_false /* don't skip invalid offsets */, + rd_false /* any offset */, rd_true /* use topic id */, + rd_false /* don't use topic name */, fields); +} + +/** + * @brief Handle ShareGroupHeartbeat request (API Key 76). + */ +static int +rd_kafka_mock_handle_ShareGroupHeartbeat(rd_kafka_mock_connection_t *mconn, + rd_kafka_buf_t *rkbuf) { + const rd_bool_t log_decode_errors = rd_true; + rd_kafka_mock_cluster_t *mcluster = mconn->broker->cluster; + rd_kafka_buf_t *resp; + rd_kafkap_str_t GroupId, MemberId, RackId; + rd_kafkap_str_t *SubscribedTopicNames = NULL; + int32_t MemberEpoch = 0, SubscribedTopicNamesCnt; + int32_t i; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_kafka_mock_sharegroup_t *mshgrp = NULL; + rd_kafka_mock_sharegroup_member_t *member = NULL; + rd_bool_t assignment_changed = rd_false; + + resp = rd_kafka_mock_buf_new_response(rkbuf); + + /* Inject Error */ + err = rd_kafka_mock_next_request_error(mconn, resp); + if (err) + goto build_response; + + /* GroupId */ + rd_kafka_buf_read_str(rkbuf, &GroupId); + + /* Coordinator check */ + { + rd_kafka_mock_broker_t *mrkb; + + mrkb = rd_kafka_mock_cluster_get_coord( + mcluster, RD_KAFKA_COORD_GROUP, &GroupId); + + if (!mrkb) + err = RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE; + else if (mrkb != mconn->broker) + err = RD_KAFKA_RESP_ERR_NOT_COORDINATOR; + } + + if (err) + goto build_response; + + /* MemberId */ + rd_kafka_buf_read_str(rkbuf, &MemberId); + + /* MemberEpoch */ + rd_kafka_buf_read_i32(rkbuf, &MemberEpoch); + + /* RackId (nullable) */ + rd_kafka_buf_read_str(rkbuf, &RackId); + + /* SubscribedTopicNames array (nullable) */ + rd_kafka_buf_read_arraycnt(rkbuf, &SubscribedTopicNamesCnt, + RD_KAFKAP_TOPICS_MAX); + if (SubscribedTopicNamesCnt >= 0) { + SubscribedTopicNames = rd_calloc( + SubscribedTopicNamesCnt > 0 ? SubscribedTopicNamesCnt : 1, + sizeof(rd_kafkap_str_t)); + for (i = 0; i < SubscribedTopicNamesCnt; i++) { + rd_kafka_buf_read_str(rkbuf, &SubscribedTopicNames[i]); + } + } + + { + mtx_lock(&mcluster->lock); + + mshgrp = rd_kafka_mock_sharegroup_get(mcluster, &GroupId); + + if (MemberEpoch == -1) { + /* LEAVE: Member wants to leave */ + member = rd_kafka_mock_sharegroup_member_find( + mshgrp, &MemberId); + if (member) { + rd_kafka_mock_sharegroup_member_destroy(mshgrp, + member); + member = NULL; + assignment_changed = rd_true; + } + + } else if (MemberEpoch == 0) { + /* JOIN: New member wants to join */ + + /* Check max group size before allowing join */ + if (mshgrp->max_size > 0 && + !rd_kafka_mock_sharegroup_member_find(mshgrp, + &MemberId) && + mshgrp->member_cnt >= mshgrp->max_size) { + err = RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED; + mtx_unlock(&mcluster->lock); + goto build_response; + } + + member = rd_kafka_mock_sharegroup_member_get( + mshgrp, &MemberId, MemberEpoch, mconn); + + if (member) { + if (rd_kafka_mock_sharegroup_member_subscribed_topic_names_set( + member, SubscribedTopicNames, + SubscribedTopicNamesCnt)) { + assignment_changed = rd_true; + } else { + /* New member always triggers + * recalculation */ + assignment_changed = rd_true; + } + MemberEpoch = member->member_epoch; + } else { + err = RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID; + } + + } else { + /* HEARTBEAT: Existing member heartbeat */ + member = rd_kafka_mock_sharegroup_member_find( + mshgrp, &MemberId); + if (!member) { + err = RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID; + } else if (MemberEpoch > member->member_epoch) { + /* Client epoch is ahead of server - indicates + * a bug or stale coordinator. */ + err = RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH; + } else if (MemberEpoch < member->member_epoch) { + /* Client epoch is behind. Allow if it matches + * the previous epoch (response with bumped + * epoch may have been lost). Otherwise fence. + */ + if (MemberEpoch != + member->previous_member_epoch) { + err = + RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH; + } else { + /* Accept previous epoch - client is + * catching up */ + member->conn = mconn; + MemberEpoch = member->member_epoch; + rd_kafka_mock_sharegroup_member_active( + mshgrp, member); + } + } else { + /* Epoch matches - normal heartbeat */ + /* Check for subscription changes */ + if (SubscribedTopicNamesCnt >= 0 && + rd_kafka_mock_sharegroup_member_subscribed_topic_names_set( + member, SubscribedTopicNames, + SubscribedTopicNamesCnt)) { + assignment_changed = rd_true; + } + member->conn = mconn; + MemberEpoch = member->member_epoch; + rd_kafka_mock_sharegroup_member_active(mshgrp, + member); + } + } + + /* Recalculate assignments if needed */ + if (assignment_changed && mshgrp->member_cnt > 0) { + rd_kafka_mock_sharegroup_assignment_recalculate(mshgrp); + if (member) + MemberEpoch = member->member_epoch; + } + + mtx_unlock(&mcluster->lock); + } + +build_response: + + /* ThrottleTimeMs */ + rd_kafka_buf_write_i32(resp, 0); + + /* ErrorCode */ + rd_kafka_buf_write_i16(resp, err); + + /* ErrorMessage */ + if (err) + rd_kafka_buf_write_str(resp, rd_kafka_err2str(err), -1); + else + rd_kafka_buf_write_str(resp, NULL, -1); + + /* MemberId */ + if (!err && member) + rd_kafka_buf_write_str(resp, member->id, -1); + else + rd_kafka_buf_write_str(resp, NULL, -1); + + /* MemberEpoch */ + rd_kafka_buf_write_i32(resp, MemberEpoch); + + /* HeartbeatIntervalMs */ + if (mshgrp) + rd_kafka_buf_write_i32(resp, mshgrp->heartbeat_interval_ms); + else + rd_kafka_buf_write_i32(resp, 5000); + + /* Assignment */ + if (!err && member && member->assignment) { + /* Send assignment even if empty (cnt == 0). + * Null (-1) means "no change", while an empty assignment + * means "you have 0 partitions". */ + rd_kafka_buf_write_i8(resp, 1); + rd_kafka_mock_handle_ShareGroupHeartbeat_write_TopicPartitions( + resp, member->assignment); + rd_kafka_buf_write_tags_empty(resp); + } else { + rd_kafka_buf_write_i8(resp, -1); + } + + rd_kafka_buf_write_tags_empty(resp); + + rd_kafka_mock_connection_send_response(mconn, resp); + + RD_IF_FREE(SubscribedTopicNames, rd_free); + return 0; + +err_parse: + RD_IF_FREE(SubscribedTopicNames, rd_free); + rd_kafka_buf_destroy(resp); + return -1; +} + +static rd_kafka_mock_sgrp_partmeta_t * +rd_kafka_mock_sgrp_partmeta_find(rd_kafka_mock_sharegroup_t *sgrp, + rd_kafka_Uuid_t topic_id, + int32_t partition) { + rd_kafka_mock_sgrp_partmeta_t *pmeta; + + TAILQ_FOREACH(pmeta, &sgrp->partitions, link) { + if (pmeta->partition != partition) + continue; + if (!rd_kafka_Uuid_cmp(pmeta->topic_id, topic_id)) + return pmeta; + } + + return NULL; +} + +static rd_kafka_mock_sgrp_partmeta_t * +rd_kafka_mock_sgrp_partmeta_get(rd_kafka_mock_sharegroup_t *sgrp, + rd_kafka_Uuid_t topic_id, + int32_t partition, + const rd_kafka_mock_partition_t *mpart) { + rd_kafka_mock_sgrp_partmeta_t *pmeta; + int64_t log_start; + int64_t log_end; + + pmeta = rd_kafka_mock_sgrp_partmeta_find(sgrp, topic_id, partition); + if (pmeta) { + log_start = mpart->start_offset; + log_end = mpart->end_offset; + if (log_start > pmeta->spso) { + /* Log retention moved start_offset past SPSO. + * Archive all in-flight records below the new + * SPSO — they are no longer in the log. */ + rd_kafka_mock_sgrp_record_state_t *state, *tmp; + TAILQ_FOREACH_SAFE(state, &pmeta->inflight, link, tmp) { + if (state->offset >= log_start) + continue; + if (state->state == + RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED) + pmeta->acquired_cnt--; + state->state = + RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED; + RD_IF_FREE(state->owner_member_id, rd_free); + state->owner_member_id = NULL; + state->lock_expiry_ts = 0; + } + pmeta->spso = log_start; + } + if (log_end > log_start) { + int64_t new_speo = log_end - 1; + if (new_speo > pmeta->speo) + pmeta->speo = new_speo; + } + return pmeta; + } + + pmeta = rd_calloc(1, sizeof(*pmeta)); + pmeta->topic_id = topic_id; + pmeta->partition = partition; + /* Initialize SPSO based on auto.offset.reset: + * 0 = latest (end of log), 1 = earliest (start of log). */ + if (sgrp->auto_offset_reset == 1) + pmeta->spso = mpart->start_offset; + else + pmeta->spso = mpart->end_offset; + if (mpart->end_offset > mpart->start_offset) + pmeta->speo = mpart->end_offset - 1; + else + pmeta->speo = mpart->start_offset - 1; + TAILQ_INIT(&pmeta->inflight); + + TAILQ_INSERT_TAIL(&sgrp->partitions, pmeta, link); + sgrp->partition_cnt++; + + return pmeta; +} + +static rd_kafka_mock_sgrp_record_state_t * +rd_kafka_mock_sgrp_record_state_find(rd_kafka_mock_sgrp_partmeta_t *pmeta, + int64_t offset) { + rd_kafka_mock_sgrp_record_state_t *state; + + TAILQ_FOREACH(state, &pmeta->inflight, link) { + if (state->offset == offset) + return state; + } + + return NULL; +} + +static rd_kafka_mock_sgrp_record_state_t * +rd_kafka_mock_sgrp_record_state_get(rd_kafka_mock_sgrp_partmeta_t *pmeta, + int64_t offset) { + rd_kafka_mock_sgrp_record_state_t *state; + + state = rd_kafka_mock_sgrp_record_state_find(pmeta, offset); + if (state) + return state; + + state = rd_calloc(1, sizeof(*state)); + state->offset = offset; + state->state = RD_KAFKA_MOCK_SGRP_RECORD_AVAILABLE; + TAILQ_INSERT_TAIL(&pmeta->inflight, state, link); + pmeta->inflight_cnt++; + + return state; +} + +static int32_t +rd_kafka_mock_msgset_est_record_size(const rd_kafka_mock_msgset_t *mset) { + int64_t record_cnt = mset->last_offset - mset->first_offset + 1; + int32_t size; + + if (record_cnt <= 0) + return 1; + + size = (int32_t)(RD_KAFKAP_BYTES_LEN(&mset->bytes) / record_cnt); + if (size <= 0) + size = 1; + + return size; +} + +/** + * @brief Check if an offset belongs to an aborted transaction on this + * partition. + * + * @locks mcluster->lock MUST be held (or single-threaded context). + */ +static rd_bool_t +rd_kafka_mock_offset_is_aborted(const rd_kafka_mock_partition_t *mpart, + int64_t offset, + int64_t pid_id) { + rd_kafka_mock_aborted_txn_t *mabort; + TAILQ_FOREACH(mabort, &mpart->aborted_txns, link) { + if (mabort->pid_id == pid_id && + offset >= mabort->first_offset && + offset <= mabort->last_offset) + return rd_true; + } + return rd_false; +} + +static void rd_kafka_mock_sgrp_acquire_available_offsets( + rd_kafka_mock_sgrp_partmeta_t *pmeta, + const rd_kafka_mock_partition_t *mpart, + const rd_kafkap_str_t *member_id, + rd_ts_t lock_expiry_ts, + int max_delivery_attempts, + int max_record_locks, + int64_t *remaining_records, + int64_t *remaining_bytes, + int *acquired_cnt, + int64_t *acquired_bytes, + int isolation_level) { + int64_t offset; + int64_t upper_bound = pmeta->speo; + + /* For read_committed, cap acquisition at LSO - 1. + * When lso == 0 (open txn starts at offset 0), upper_bound + * becomes -1, which makes the loop body unreachable — + * correct behavior: block all reads until the txn resolves. */ + if (isolation_level == 1 && mpart->lso - 1 < upper_bound) + upper_bound = mpart->lso - 1; + + for (offset = pmeta->spso; offset <= upper_bound; offset++) { + const rd_kafka_mock_msgset_t *mset; + rd_kafka_mock_sgrp_record_state_t *state; + int32_t est_size; + + if (remaining_records && *remaining_records == 0) + break; + if (remaining_bytes && *remaining_bytes == 0) + break; + + /* Check max acquired record locks per partition */ + if (max_record_locks > 0 && + pmeta->acquired_cnt >= max_record_locks) + break; + + state = rd_kafka_mock_sgrp_record_state_find(pmeta, offset); + if (state && + state->state != RD_KAFKA_MOCK_SGRP_RECORD_AVAILABLE) + continue; + + /* Check max delivery attempts: if the record has already + * been acquired (and released/expired) too many times, + * archive it instead of re-acquiring. */ + if (max_delivery_attempts > 0 && state && + state->delivery_count >= max_delivery_attempts) { + state->state = RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED; + RD_IF_FREE(state->owner_member_id, rd_free); + state->owner_member_id = NULL; + state->lock_expiry_ts = 0; + continue; + } + + mset = rd_kafka_mock_msgset_find(mpart, offset, rd_false); + if (!mset) + continue; + + /* For read_committed: skip aborted transactional data batches + * by archiving them immediately. */ + if (isolation_level == 1) { + int16_t attrs = 0; + int64_t batch_pid = 0; + memcpy(&attrs, + (const char *)mset->bytes.data + + RD_KAFKAP_MSGSET_V2_OF_Attributes, + sizeof(attrs)); + attrs = be16toh(attrs); + if (attrs & RD_KAFKA_MSGSET_V2_ATTR_TRANSACTIONAL) { + memcpy(&batch_pid, + (const char *)mset->bytes.data + + RD_KAFKAP_MSGSET_V2_OF_ProducerId, + sizeof(batch_pid)); + batch_pid = be64toh(batch_pid); + if (!(attrs & + RD_KAFKA_MSGSET_V2_ATTR_CONTROL) && + rd_kafka_mock_offset_is_aborted( + mpart, offset, batch_pid)) { + /* Archive aborted data batch */ + state = + rd_kafka_mock_sgrp_record_state_get( + pmeta, offset); + state->state = + RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED; + RD_IF_FREE(state->owner_member_id, + rd_free); + state->owner_member_id = NULL; + state->lock_expiry_ts = 0; + continue; + } + } + } + + est_size = rd_kafka_mock_msgset_est_record_size(mset); + if (remaining_bytes && *remaining_bytes > 0 && + est_size > *remaining_bytes) + break; + + state = rd_kafka_mock_sgrp_record_state_get(pmeta, offset); + state->state = RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED; + state->lock_expiry_ts = lock_expiry_ts; + state->delivery_count++; + RD_IF_FREE(state->owner_member_id, rd_free); + state->owner_member_id = RD_KAFKAP_STR_DUP(member_id); + pmeta->acquired_cnt++; + + (*acquired_cnt)++; + *acquired_bytes += est_size; + if (remaining_records && *remaining_records > 0) + (*remaining_records)--; + if (remaining_bytes && *remaining_bytes > 0) + (*remaining_bytes) -= est_size; + } +} + +static void rd_kafka_mock_sgrp_partmeta_prune_archived( + rd_kafka_mock_sgrp_partmeta_t *pmeta) { + rd_kafka_mock_sgrp_record_state_t *state, *tmp; + + TAILQ_FOREACH_SAFE(state, &pmeta->inflight, link, tmp) { + if (state->state != RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED && + state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACKNOWLEDGED) + continue; + if (state->offset >= pmeta->spso) + continue; + + TAILQ_REMOVE(&pmeta->inflight, state, link); + pmeta->inflight_cnt--; + rd_free(state->owner_member_id); + rd_free(state); + } +} + +/** + * @brief Write all acquired record batches for the given partition and member + * into the response buffer as a single Records field (compact bytes + * containing concatenated RecordBatches). + * + * @returns the total number of record data bytes written (0 if none). + */ +static size_t rd_kafka_mock_sgrp_write_acquired_records( + rd_kafka_buf_t *resp, + rd_kafka_mock_sgrp_partmeta_t *pmeta, + const rd_kafka_mock_partition_t *mpart, + const rd_kafkap_str_t *member_id, + rd_ts_t now) { + rd_list_t msgsets; + int64_t offset; + size_t total_len = 0; + int i; + + rd_list_init(&msgsets, 16, NULL); /* no free_cb: borrowed ptrs */ + + /* Collect unique msgsets containing acquired records for this member */ + for (offset = pmeta->spso; offset <= pmeta->speo; offset++) { + rd_kafka_mock_sgrp_record_state_t *state = + rd_kafka_mock_sgrp_record_state_find(pmeta, offset); + const rd_kafka_mock_msgset_t *mset; + + if (!state || + state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED) + continue; + + if (state->lock_expiry_ts && state->lock_expiry_ts <= now) + continue; + + if (rd_kafkap_str_cmp_str(member_id, state->owner_member_id)) + continue; + + mset = rd_kafka_mock_msgset_find(mpart, offset, rd_false); + if (!mset) + continue; + + /* Deduplicate: multiple offsets may fall in the same batch */ + if (rd_list_find(&msgsets, mset, rd_list_cmp_ptr)) + continue; + + rd_list_add(&msgsets, (void *)mset); + total_len += RD_KAFKAP_BYTES_LEN(&mset->bytes); + } + + if (rd_list_cnt(&msgsets) == 0) { + /* No acquired records: write empty compact bytes + * (uvarint(1) = 0 data bytes). Do NOT write NULL + * (uvarint(0)) because the client reads this field + * with rd_kafka_buf_read_arraycnt() which decodes + * NULL as -1 and triggers a parse failure. */ + rd_list_destroy(&msgsets); + rd_kafka_buf_write_uvarint(resp, 1); + return 0; + } + + /* Write compact bytes length prefix (N+1 encoding) */ + rd_kafka_buf_write_uvarint(resp, (uint64_t)(total_len + 1)); + + /* Write each msgset's raw bytes back-to-back */ + for (i = 0; i < rd_list_cnt(&msgsets); i++) { + const rd_kafka_mock_msgset_t *mset = rd_list_elem(&msgsets, i); + rd_kafka_buf_write(resp, mset->bytes.data, + RD_KAFKAP_BYTES_LEN(&mset->bytes)); + } + + rd_list_destroy(&msgsets); + return total_len; +} + +/** + * @brief Temporary structure for accumulating acknowledgement batches + * during ShareFetch request parsing. + */ +struct rd_kafka_mock_sgrp_ack_entry { + rd_kafka_Uuid_t topic_id; + int32_t partition; + int64_t first_offset; + int64_t last_offset; + int8_t ack_type; /**< 0=GAP, 1=ACCEPT, 2=RELEASE, 3=REJECT */ + rd_kafka_resp_err_t err; /**< Per-batch ack result, set after apply */ +}; + +/** + * @brief Allocate and initialize a new ack entry. + */ +static struct rd_kafka_mock_sgrp_ack_entry * +rd_kafka_mock_sgrp_ack_entry_new(rd_kafka_Uuid_t topic_id, + int32_t partition, + int64_t first_offset, + int64_t last_offset, + int8_t ack_type) { + struct rd_kafka_mock_sgrp_ack_entry *entry = + rd_calloc(1, sizeof(*entry)); + entry->topic_id = topic_id; + entry->partition = partition; + entry->first_offset = first_offset; + entry->last_offset = last_offset; + entry->ack_type = ack_type; + return entry; +} + +/** + * @brief Find the first ack error for the given (topic_id, partition) across + * all ack entries. Returns NO_ERROR if no ack targeted this partition + * or all acks succeeded. + */ +static rd_kafka_resp_err_t +rd_kafka_mock_sgrp_ack_error_for_partition(const rd_list_t *ack_entries, + rd_kafka_Uuid_t topic_id, + int32_t partition) { + int k; + for (k = 0; k < rd_list_cnt(ack_entries); k++) { + const struct rd_kafka_mock_sgrp_ack_entry *entry = + rd_list_elem(ack_entries, k); + if (entry->partition == partition && + !rd_kafka_Uuid_cmp(entry->topic_id, topic_id) && entry->err) + return entry->err; + } + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +/** + * @brief Apply a single acknowledgement batch to share-group partition + * metadata. + * + * For GAP (0), ACCEPT (1) and REJECT (3): mark records as ARCHIVED. + * For RELEASE (2): mark records as AVAILABLE. + * + * After applying, advance SPSO past contiguous ARCHIVED records so that + * the share-partition start offset moves forward. + * + * @locks mcluster->lock MUST be held. + */ +static rd_kafka_resp_err_t +rd_kafka_mock_sgrp_apply_ack(rd_kafka_mock_sharegroup_t *sgrp, + rd_kafka_Uuid_t topic_id, + int32_t partition, + int64_t first_offset, + int64_t last_offset, + int8_t ack_type, + const rd_kafkap_str_t *member_id) { + rd_kafka_mock_sgrp_partmeta_t *pmeta; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; + int64_t offset; + + pmeta = rd_kafka_mock_sgrp_partmeta_find(sgrp, topic_id, partition); + if (!pmeta) + return RD_KAFKA_RESP_ERR_NO_ERROR; + + /* Pass 1: Validate all offsets in the range. + * Acks for a partition in a single request are atomic — + * all succeed or all fail together. */ + for (offset = first_offset; offset <= last_offset; offset++) { + rd_kafka_mock_sgrp_record_state_t *state; + + /* Offsets below SPSO are already logically archived + * (e.g. due to log retention). Acks for such + * records are silently accepted — no error is + * thrown for a record that is about to become + * Archived. */ + if (offset < pmeta->spso) + continue; + + state = rd_kafka_mock_sgrp_record_state_find(pmeta, offset); + if (!state) + continue; + + /* Only the owning member may acknowledge acquired records. + * If the record's lock expired (reverted to Available), + * was re-acquired by another member, or is otherwise not + * in ACQUIRED state for this member, report + * INVALID_RECORD_STATE. */ + if (state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED || + !state->owner_member_id || + rd_kafkap_str_cmp_str(member_id, state->owner_member_id)) { + err = RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE; + break; + } + } + + /* Pass 2: Apply state transitions only if validation passed. */ + if (!err) { + for (offset = first_offset; offset <= last_offset; offset++) { + rd_kafka_mock_sgrp_record_state_t *state; + + if (offset < pmeta->spso) + continue; + + state = + rd_kafka_mock_sgrp_record_state_find(pmeta, offset); + if (!state) + continue; + + switch (ack_type) { + case 1: /* ACCEPT -> Acknowledged */ + pmeta->acquired_cnt--; + state->state = + RD_KAFKA_MOCK_SGRP_RECORD_ACKNOWLEDGED; + rd_free(state->owner_member_id); + state->owner_member_id = NULL; + state->lock_expiry_ts = 0; + break; + case 0: /* GAP -> Archived */ + case 3: /* REJECT -> Archived */ + pmeta->acquired_cnt--; + state->state = + RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED; + rd_free(state->owner_member_id); + state->owner_member_id = NULL; + state->lock_expiry_ts = 0; + break; + case 2: /* RELEASE */ + rd_kafka_mock_sgrp_record_release(sgrp, pmeta, + state); + break; + default: + break; + } + } + } + + /* Advance SPSO past contiguous ACKNOWLEDGED or ARCHIVED records + * from the start, transitioning ACKNOWLEDGED to ARCHIVED. */ + while (pmeta->spso <= pmeta->speo) { + rd_kafka_mock_sgrp_record_state_t *state = + rd_kafka_mock_sgrp_record_state_find(pmeta, pmeta->spso); + if (!state || + (state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACKNOWLEDGED && + state->state != RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED)) + break; + state->state = RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED; + pmeta->spso++; + } + + return err; +} + +/** + * @brief Write AcquiredRecords metadata for the given partition and member + * into the response buffer. + * + * Groups contiguous ACQUIRED records (owned by this member and not expired) + * into ranges and writes each range as (FirstOffset, LastOffset, + * DeliveryCount). + * + * @locks mcluster->lock MUST be held. + */ +static void rd_kafka_mock_sgrp_write_acquired_records_meta( + rd_kafka_buf_t *resp, + rd_kafka_mock_sgrp_partmeta_t *pmeta, + const rd_kafkap_str_t *member_id, + rd_ts_t now) { + struct acquired_range { + int64_t first_offset; + int64_t last_offset; + int16_t delivery_count; + }; + rd_list_t ranges; + int64_t offset; + int i; + + rd_list_init(&ranges, 16, rd_free); + + for (offset = pmeta->spso; offset <= pmeta->speo; offset++) { + rd_kafka_mock_sgrp_record_state_t *state = + rd_kafka_mock_sgrp_record_state_find(pmeta, offset); + struct acquired_range *cur; + + if (!state || + state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED || + !state->owner_member_id || + rd_kafkap_str_cmp_str(member_id, state->owner_member_id) || + (state->lock_expiry_ts && state->lock_expiry_ts <= now)) + continue; + + /* Try to extend the current range */ + if (rd_list_cnt(&ranges) > 0) { + cur = rd_list_elem(&ranges, rd_list_cnt(&ranges) - 1); + if (offset == cur->last_offset + 1 && + (int16_t)state->delivery_count == + cur->delivery_count) { + cur->last_offset = offset; + continue; + } + } + + /* Start a new range */ + cur = rd_calloc(1, sizeof(*cur)); + cur->first_offset = offset; + cur->last_offset = offset; + cur->delivery_count = (int16_t)state->delivery_count; + rd_list_add(&ranges, cur); + } + + rd_kafka_buf_write_arraycnt(resp, rd_list_cnt(&ranges)); + + for (i = 0; i < rd_list_cnt(&ranges); i++) { + struct acquired_range *r = rd_list_elem(&ranges, i); + rd_kafka_buf_write_i64(resp, r->first_offset); + rd_kafka_buf_write_i64(resp, r->last_offset); + rd_kafka_buf_write_i16(resp, r->delivery_count); + rd_kafka_buf_write_tags_empty(resp); + } + + rd_list_destroy(&ranges); +} + +static int rd_kafka_mock_handle_ShareFetch(rd_kafka_mock_connection_t *mconn, + rd_kafka_buf_t *rkbuf) { + const rd_bool_t log_decode_errors = rd_true; + rd_kafka_mock_cluster_t *mcluster = mconn->broker->cluster; + rd_kafka_buf_t *resp = rd_kafka_mock_buf_new_response(rkbuf); + rd_kafkap_str_t GroupId, MemberId; + int32_t SessionEpoch = -1, MaxWaitMs = 0, MinBytes = 0, MaxBytes = 0, + MaxRecords = 0, BatchSize = 0; + int32_t TopicsCnt; + int32_t ForgottenTopicsCnt; + rd_kafka_topic_partition_list_t *requested_partitions = NULL; + rd_kafka_topic_partition_list_t *forgotten_partitions = NULL; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_bool_t ack_parse_err = rd_false; + rd_kafka_mock_sharegroup_t *sgrp = NULL; + rd_kafka_mock_sgrp_fetch_session_t *session = NULL; + rd_list_t ack_entries; + rd_kafka_mock_broker_t *node_endpoints[64]; + int node_endpoint_cnt = 0; + int k; + + (void)log_decode_errors; + + rd_list_init(&ack_entries, 8, rd_free); + + rd_kafka_buf_read_str(rkbuf, &GroupId); + rd_kafka_buf_read_str(rkbuf, &MemberId); + /* ShareFetch has ShareSessionEpoch only, no SessionId. + * Sessions are keyed by (GroupId, MemberId, NodeId). */ + rd_kafka_buf_read_i32(rkbuf, &SessionEpoch); + rd_kafka_buf_read_i32(rkbuf, &MaxWaitMs); + rd_kafka_buf_read_i32(rkbuf, &MinBytes); + rd_kafka_buf_read_i32(rkbuf, &MaxBytes); + rd_kafka_buf_read_i32(rkbuf, &MaxRecords); + rd_kafka_buf_read_i32(rkbuf, &BatchSize); + + requested_partitions = rd_kafka_topic_partition_list_new(0); + + rd_kafka_buf_read_arraycnt(rkbuf, &TopicsCnt, RD_KAFKAP_TOPICS_MAX); + while (TopicsCnt-- > 0) { + rd_kafka_Uuid_t TopicId = RD_KAFKA_UUID_ZERO; + int32_t PartitionCnt; + + rd_kafka_buf_read_uuid(rkbuf, &TopicId); + rd_kafka_buf_read_arraycnt(rkbuf, &PartitionCnt, + RD_KAFKAP_PARTITIONS_MAX); + + while (PartitionCnt-- > 0) { + int32_t Partition; + int32_t AckBatchCnt; + int64_t prev_ack_last = -1; + rd_kafka_topic_partition_t *rktpar; + + rd_kafka_buf_read_i32(rkbuf, &Partition); + rd_kafka_buf_read_arraycnt(rkbuf, &AckBatchCnt, -1); + while (AckBatchCnt-- > 0) { + int32_t AckTypeCnt; + int64_t AckFirstOffset, AckLastOffset; + int64_t range_len, ti; + int8_t *ack_types = NULL; + int8_t single_type = -1; + + rd_kafka_buf_read_i64(rkbuf, &AckFirstOffset); + rd_kafka_buf_read_i64(rkbuf, &AckLastOffset); + + /* Validate ascending order and + * non-overlapping ranges. */ + if (prev_ack_last >= 0 && + AckFirstOffset <= prev_ack_last) + ack_parse_err = rd_true; + prev_ack_last = AckLastOffset; + + range_len = AckLastOffset - AckFirstOffset + 1; + + rd_kafka_buf_read_arraycnt(rkbuf, &AckTypeCnt, + -1); + + if (AckTypeCnt == 1) { + /* Single type for entire range */ + rd_kafka_buf_read_i8(rkbuf, + &single_type); + } else if (AckTypeCnt > 1) { + /* Per-offset types */ + ack_types = + rd_alloca((size_t)AckTypeCnt * + sizeof(*ack_types)); + for (ti = 0; ti < AckTypeCnt; ti++) + rd_kafka_buf_read_i8( + rkbuf, &ack_types[ti]); + } + rd_kafka_buf_skip_tags(rkbuf); + + if (AckTypeCnt == 1 && single_type >= 0) { + rd_list_add( + &ack_entries, + rd_kafka_mock_sgrp_ack_entry_new( + TopicId, Partition, + AckFirstOffset, AckLastOffset, + single_type)); + } else if (ack_types && + AckTypeCnt == range_len) { + for (ti = 0; ti < range_len; ti++) { + rd_list_add( + &ack_entries, + rd_kafka_mock_sgrp_ack_entry_new( + TopicId, Partition, + AckFirstOffset + ti, + AckFirstOffset + ti, + ack_types[ti])); + } + } else if (AckTypeCnt > 0) { + ack_parse_err = rd_true; + } + } + + rktpar = rd_kafka_topic_partition_list_add( + requested_partitions, "", Partition); + rd_kafka_topic_partition_set_topic_id(rktpar, TopicId); + + rd_kafka_buf_skip_tags(rkbuf); + } + + rd_kafka_buf_skip_tags(rkbuf); + } + + rd_kafka_buf_read_arraycnt(rkbuf, &ForgottenTopicsCnt, + RD_KAFKAP_TOPICS_MAX); + if (ForgottenTopicsCnt > 0) + forgotten_partitions = + rd_kafka_topic_partition_list_new(ForgottenTopicsCnt); + while (ForgottenTopicsCnt-- > 0) { + rd_kafka_Uuid_t ForgTopicId = RD_KAFKA_UUID_ZERO; + int32_t ForgPartCnt; + rd_kafka_buf_read_uuid(rkbuf, &ForgTopicId); + rd_kafka_buf_read_arraycnt(rkbuf, &ForgPartCnt, + RD_KAFKAP_PARTITIONS_MAX); + while (ForgPartCnt-- > 0) { + int32_t ForgPartition; + rd_kafka_topic_partition_t *ftp; + rd_kafka_buf_read_i32(rkbuf, &ForgPartition); + + /* Record in forgotten list for session + * removal and inflight release. */ + ftp = rd_kafka_topic_partition_list_add( + forgotten_partitions, "", ForgPartition); + rd_kafka_topic_partition_set_topic_id(ftp, ForgTopicId); + + /* Remove from requested_partitions so they + * are not fetched in this request. */ + if (requested_partitions) { + int idx = + rd_kafka_topic_partition_list_find_idx_by_id( + requested_partitions, ForgTopicId, + ForgPartition); + if (idx >= 0) + rd_kafka_topic_partition_list_del_by_idx( + requested_partitions, idx); + } + } + /* ForgottenTopic tags */ + rd_kafka_buf_skip_tags(rkbuf); + } + + /* Top-level tags */ + rd_kafka_buf_skip_tags(rkbuf); + + rd_kafka_dbg(mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareFetch parsed: group %.*s member %.*s " + "session_epoch %" PRId32 " max_wait %" PRId32 + " min_bytes %" PRId32 " max_bytes %" PRId32 + " max_records %" PRId32 " batch_size %" PRId32, + RD_KAFKAP_STR_PR(&GroupId), RD_KAFKAP_STR_PR(&MemberId), + SessionEpoch, MaxWaitMs, MinBytes, MaxBytes, MaxRecords, + BatchSize); + + err = rd_kafka_mock_next_request_error(mconn, resp); + + if (!err && ack_parse_err) + err = RD_KAFKA_RESP_ERR_INVALID_REQUEST; + + if (!err) { + int64_t remaining_records = + MaxRecords > 0 ? (int64_t)MaxRecords : -1; + int64_t remaining_bytes = MaxBytes > 0 ? (int64_t)MaxBytes : -1; + int acquired_cnt = 0; + int64_t acquired_bytes = 0; + rd_ts_t now = rd_clock(); + + /* Session management. + * Sessions are keyed by (GroupId, MemberId). + * SessionEpoch: 0 = open new session, + * -1 = close session, + * >0 = continue (must match expected epoch). + * + * session->partitions is the source of truth for the + * full partition set. For epoch=0, it is set from the + * request. For epoch>0, the request only carries + * additions; an empty list means "no changes". + * Forgotten partitions are removed below. + * Acquisition and response writing always use + * session->partitions. */ + mtx_lock(&mcluster->lock); + sgrp = rd_kafka_mock_sharegroup_get(mcluster, &GroupId); + + /* epoch=0 (full fetch / new session) must not + * contain acknowledgements. Check BEFORE + * session_validate to avoid destroying an existing + * session for a malformed request. */ + if (SessionEpoch == 0 && rd_list_cnt(&ack_entries) > 0) { + rd_kafka_dbg(mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareFetch: rejecting epoch=0 request " + "with %d ack(s) (INVALID_REQUEST)", + rd_list_cnt(&ack_entries)); + err = RD_KAFKA_RESP_ERR_INVALID_REQUEST; + } + + /* Common validation: member check, session lookup, + * epoch -1 close, epoch > 0 validation. */ + if (!err) + err = rd_kafka_mock_sgrp_session_validate( + sgrp, &MemberId, mconn->broker->id, SessionEpoch, + &session, "ShareFetch"); + + if (!err && SessionEpoch == 0) { + /* Open a new session (or reuse if one already exists + * for this member on this broker). */ + int broker_session_cnt = 0; + rd_kafka_mock_sharegroup_t *sg; + rd_kafka_mock_sgrp_fetch_session_t *s; + /* Count sessions across ALL share groups on this + * broker. group.share.max.share.sessions is a + * per-broker limit with cache key (GroupId, + * MemberId). */ + TAILQ_FOREACH(sg, &mcluster->sharegrps, link) { + TAILQ_FOREACH(s, &sg->fetch_sessions, link) { + if (s->node_id == mconn->broker->id) + broker_session_cnt++; + } + } + if (!session && sgrp->max_fetch_sessions > 0 && + broker_session_cnt >= sgrp->max_fetch_sessions) { + /* Session cache is full for this broker. */ + err = + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED; + } else if (!session) { + session = rd_calloc(1, sizeof(*session)); + session->member_id = + RD_KAFKAP_STR_DUP(&MemberId); + session->node_id = mconn->broker->id; + session->session_epoch = 0; + session->partitions = + rd_kafka_topic_partition_list_copy( + requested_partitions); + TAILQ_INSERT_TAIL(&sgrp->fetch_sessions, + session, link); + sgrp->fetch_session_cnt++; + } else { + /* Session already exists for this member; + * reset it. */ + session->session_epoch = 0; + RD_IF_FREE( + session->partitions, + rd_kafka_topic_partition_list_destroy); + session->partitions = + rd_kafka_topic_partition_list_copy( + requested_partitions); + } + } else if (!err && session && SessionEpoch > 0) { + /* Session continuation: merge additions from the + * request into session->partitions. An empty + * request list means "no changes". */ + if (requested_partitions && + requested_partitions->cnt > 0) { + rd_kafka_topic_partition_t *rp; + if (!session->partitions) + session->partitions = + rd_kafka_topic_partition_list_new( + requested_partitions->cnt); + RD_KAFKA_TPLIST_FOREACH(rp, + requested_partitions) { + rd_kafka_Uuid_t tid = + rd_kafka_topic_partition_get_topic_id( + rp); + if (rd_kafka_topic_partition_list_find_idx_by_id( + session->partitions, tid, + rp->partition) < 0) { + rd_kafka_topic_partition_t *np = + rd_kafka_topic_partition_list_add( + session->partitions, "", + rp->partition); + rd_kafka_topic_partition_set_topic_id( + np, tid); + } + } + } + } + + /* For all successful, non-close requests: update activity + * timestamp and increment epoch for next request. */ + if (!err && session && SessionEpoch != -1) { + session->ts_last_activity = rd_clock(); + session->session_epoch++; + } + + /* epoch=-1 (final fetch / close session) must not + * contain ForgottenTopicsData. Partitions + * in the Topics array ARE allowed — they carry + * piggybacked acks for the final request. */ + if (!err && SessionEpoch == -1 && + (forgotten_partitions && forgotten_partitions->cnt > 0)) { + rd_kafka_dbg( + mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareFetch: rejecting epoch=-1 request " + "with ForgottenTopicsData (INVALID_REQUEST)"); + err = RD_KAFKA_RESP_ERR_INVALID_REQUEST; + } + + /* Apply piggy-backed acknowledgements BEFORE forgotten + * partition processing, so that acks for partitions + * being removed are applied while the records are still + * in ACQUIRED state. */ + if (!err && sgrp && rd_list_cnt(&ack_entries) > 0) { + int k; + rd_kafka_dbg(mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareFetch: applying %d acknowledgement " + "batch(es) for member %.*s", + rd_list_cnt(&ack_entries), + RD_KAFKAP_STR_PR(&MemberId)); + for (k = 0; k < rd_list_cnt(&ack_entries); k++) { + struct rd_kafka_mock_sgrp_ack_entry *entry = + rd_list_elem(&ack_entries, k); + entry->err = rd_kafka_mock_sgrp_apply_ack( + sgrp, entry->topic_id, entry->partition, + entry->first_offset, entry->last_offset, + entry->ack_type, &MemberId); + } + } + + /* Remove forgotten partitions from session and release + * any remaining ACQUIRED records owned by this member. + * Runs AFTER ack application so that acks for partitions + * being removed have already been processed. */ + if (!err && session && forgotten_partitions && + forgotten_partitions->cnt > 0) { + rd_kafka_topic_partition_t *ftp; + RD_KAFKA_TPLIST_FOREACH(ftp, forgotten_partitions) { + rd_kafka_Uuid_t ftid = + rd_kafka_topic_partition_get_topic_id(ftp); + rd_kafka_mock_sgrp_partmeta_t *pmeta; + + /* Remove from session partition list */ + if (session->partitions) { + int idx = + rd_kafka_topic_partition_list_find_idx_by_id( + session->partitions, ftid, + ftp->partition); + if (idx >= 0) + rd_kafka_topic_partition_list_del_by_idx( + session->partitions, idx); + } + + /* Release ACQUIRED records for this + * member on this partition, but only + * if this broker currently leads it: + * forgetting a partition that moved to + * another leader must not release the + * locks now held through the new + * leader's session. */ + rd_kafka_mock_topic_t *ftopic = + rd_kafka_mock_topic_find_by_id(mcluster, + ftid); + rd_kafka_mock_partition_t *fpart = + ftopic ? rd_kafka_mock_partition_find( + ftopic, ftp->partition) + : NULL; + if (!fpart || fpart->leader != mconn->broker) + continue; + + pmeta = rd_kafka_mock_sgrp_partmeta_find( + sgrp, ftid, ftp->partition); + if (pmeta) { + rd_kafka_mock_sgrp_record_state_t + *state, + *tmp; + TAILQ_FOREACH_SAFE(state, + &pmeta->inflight, + link, tmp) { + if (state->state != + RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED) + continue; + if (!state->owner_member_id) + continue; + if (rd_kafkap_str_cmp_str( + &MemberId, + state->owner_member_id)) + continue; + rd_kafka_mock_sgrp_record_release( + sgrp, pmeta, state); + } + } + } + } + + /* epoch=-1 is a final fetch: no new records are + * acquired ("No data will be fetched"). */ + if (!err && SessionEpoch != -1 && sgrp && session && + session->partitions && session->partitions->cnt > 0) { + int pi, pcnt = session->partitions->cnt; + int start = session->partition_start_idx % pcnt; + + for (pi = 0; pi < pcnt; pi++) { + int idx = (start + pi) % pcnt; + rd_kafka_topic_partition_t *rktpar = + &session->partitions->elems[idx]; + rd_kafka_Uuid_t topic_id = + rd_kafka_topic_partition_get_topic_id( + rktpar); + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find_by_id(mcluster, + topic_id); + rd_kafka_mock_partition_t *mpart; + + if (!mtopic || + !(mpart = rd_kafka_mock_partition_find( + mtopic, rktpar->partition))) { + /* Per-partition error: skip this + * partition but continue with others. + * The response writer handles the + * error via mpart==NULL check. */ + continue; + } + + /* Skip acquisition if this broker is + * not the partition leader. The + * response writer returns + * NOT_LEADER_OR_FOLLOWER. */ + if (mpart->leader != mconn->broker) + continue; + + /* Injected partition error: skip acquisition + * so no records are locked for a partition + * the client will see as failed. The error + * is stashed on the session's partition list + * element and consumed (read and cleared) by + * the response writer below. */ + rktpar->err = + rd_kafka_mock_partition_next_request_error( + mpart, rkbuf->rkbuf_reqhdr.ApiKey); + if (rktpar->err) + continue; + + rd_kafka_mock_sgrp_partmeta_t *pmeta = + rd_kafka_mock_sgrp_partmeta_get( + sgrp, topic_id, rktpar->partition, + mpart); + rd_kafka_mock_sgrp_partmeta_prune_archived( + pmeta); + rd_kafka_mock_sgrp_acquire_available_offsets( + pmeta, mpart, &MemberId, + now + ((sgrp->record_lock_duration_ms > 0 + ? sgrp->record_lock_duration_ms + : sgrp->session_timeout_ms) * + 1000), + sgrp->max_delivery_attempts, + sgrp->max_record_locks, + MaxRecords > 0 ? &remaining_records : NULL, + MaxBytes > 0 ? &remaining_bytes : NULL, + &acquired_cnt, &acquired_bytes, + sgrp->isolation_level); + } + + /* Rotate start index for next request */ + session->partition_start_idx = (start + 1) % pcnt; + } + + rd_kafka_dbg(mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareFetch acquired: %d records, %" PRId64 + " bytes", + acquired_cnt, acquired_bytes); + + /* Response: ThrottleTimeMs */ + rd_kafka_buf_write_i32(resp, 0); + /* Response: ErrorCode */ + rd_kafka_buf_write_i16(resp, err); + /* Response: ErrorMessage */ + if (err) + rd_kafka_buf_write_str(resp, rd_kafka_err2str(err), -1); + else + rd_kafka_buf_write_str(resp, NULL, -1); + /* Response: AcquisitionLockTimeoutMs — use the effective + * lock duration (same logic as the acquisition path). */ + rd_kafka_buf_write_i32( + resp, sgrp ? (sgrp->record_lock_duration_ms > 0 + ? sgrp->record_lock_duration_ms + : sgrp->session_timeout_ms) + : 0); + + if (session && session->partitions) + rd_kafka_topic_partition_list_sort_by_topic_id( + session->partitions); + + { + int i = 0; + int topic_cnt = 0; + rd_kafka_Uuid_t current_topic = RD_KAFKA_UUID_ZERO; + + for (i = 0; session && session->partitions && + i < session->partitions->cnt; + i++) { + rd_kafka_topic_partition_t *rktpar = + &session->partitions->elems[i]; + rd_kafka_Uuid_t topic_id = + rd_kafka_topic_partition_get_topic_id( + rktpar); + if (i == 0 || + rd_kafka_Uuid_cmp(topic_id, + current_topic) != 0) { + topic_cnt++; + current_topic = topic_id; + } + } + + /* Response: #Topics */ + rd_kafka_buf_write_arraycnt(resp, topic_cnt); + + i = 0; + while (session && session->partitions && + i < session->partitions->cnt) { + int j; + rd_kafka_Uuid_t topic_id = + rd_kafka_topic_partition_get_topic_id( + &session->partitions->elems[i]); + int part_cnt = 0; + + for (j = i; j < session->partitions->cnt; j++) { + rd_kafka_Uuid_t next_topic_id = + rd_kafka_topic_partition_get_topic_id( + &session->partitions->elems[j]); + if (rd_kafka_Uuid_cmp( + topic_id, next_topic_id) != 0) + break; + part_cnt++; + } + + /* Response: TopicId */ + rd_kafka_buf_write_uuid(resp, &topic_id); + /* Response: #Partitions */ + rd_kafka_buf_write_arraycnt(resp, part_cnt); + + for (j = i; j < i + part_cnt; j++) { + rd_kafka_topic_partition_t *rktpar = + &session->partitions->elems[j]; + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find_by_id( + mcluster, topic_id); + rd_kafka_mock_partition_t *mpart = + mtopic + ? rd_kafka_mock_partition_find( + mtopic, rktpar->partition) + : NULL; + rd_kafka_mock_sgrp_partmeta_t *pmeta = + mpart + ? rd_kafka_mock_sgrp_partmeta_find( + sgrp, topic_id, + rktpar->partition) + : NULL; + rd_kafka_resp_err_t ack_err; + rd_kafka_resp_err_t part_err; + + if (!mpart) + part_err = + RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + else if (mpart->leader != mconn->broker) + part_err = + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER; + else + part_err = rktpar->err; + + /* Consume the stashed injected error + * so it doesn't leak into subsequent + * responses. */ + rktpar->err = + RD_KAFKA_RESP_ERR_NO_ERROR; + + /* Response: Partition */ + rd_kafka_buf_write_i32( + resp, rktpar->partition); + /* Response: PartitionFetchErrorCode */ + rd_kafka_buf_write_i16(resp, part_err); + /* Response: PartitionFetchErrorString + */ + if (part_err) + rd_kafka_buf_write_str( + resp, + rd_kafka_err2str(part_err), + -1); + else + rd_kafka_buf_write_str( + resp, NULL, -1); + /* Response: AcknowledgementErrorCode */ + ack_err = + rd_kafka_mock_sgrp_ack_error_for_partition( + &ack_entries, topic_id, + rktpar->partition); + rd_kafka_buf_write_i16(resp, ack_err); + /* Response: + * AcknowledgementErrorString */ + if (ack_err) + rd_kafka_buf_write_str( + resp, + rd_kafka_err2str(ack_err), + -1); + else + rd_kafka_buf_write_str( + resp, NULL, -1); + /* Response: CurrentLeader. + * Populated only when the + * per-partition error signals stale + * leader info. */ + if (mpart && mpart->leader && + (part_err == + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER || + part_err == + RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH)) { + int found = 0; + rd_kafka_buf_write_i32( + resp, mpart->leader->id); + rd_kafka_buf_write_i32( + resp, mpart->leader_epoch); + for (k = 0; + k < node_endpoint_cnt; + k++) { + if (node_endpoints[k] == + mpart->leader) { + found = 1; + break; + } + } + if (!found && + node_endpoint_cnt < 64) + node_endpoints + [node_endpoint_cnt++] = + mpart->leader; + } else { + rd_kafka_buf_write_i32(resp, + -1); + rd_kafka_buf_write_i32(resp, + -1); + } + rd_kafka_buf_write_tags_empty(resp); + /* Response: Records (all acquired + * batches concatenated) */ + if (mpart && pmeta && !part_err) + rd_kafka_mock_sgrp_write_acquired_records( + resp, pmeta, mpart, + &MemberId, now); + else + rd_kafka_buf_write_uvarint(resp, + 1); + /* Response: AcquiredRecords */ + if (mpart && pmeta && !part_err) + rd_kafka_mock_sgrp_write_acquired_records_meta( + resp, pmeta, &MemberId, + now); + else + rd_kafka_buf_write_arraycnt( + resp, 0); + /* Response: Partition tags */ + rd_kafka_buf_write_tags_empty(resp); + } + + /* Response: Topic tags */ + rd_kafka_buf_write_tags_empty(resp); + + i += part_cnt; + } + } + + /* Response: NodeEndpoints */ + rd_kafka_buf_write_arraycnt(resp, node_endpoint_cnt); + for (k = 0; k < node_endpoint_cnt; k++) { + rd_kafka_mock_broker_t *nb = node_endpoints[k]; + rd_kafka_buf_write_i32(resp, nb->id); + rd_kafka_buf_write_str(resp, nb->advertised_listener, + -1); + rd_kafka_buf_write_i32(resp, (int32_t)nb->port); + rd_kafka_buf_write_str(resp, nb->rack, -1); + rd_kafka_buf_write_tags_empty(resp); + } + /* Response: Top-level tags */ + rd_kafka_buf_write_tags_empty(resp); + + /* epoch=-1 (final fetch) → release remaining acquired + * records and close the session. */ + if (!err && session && SessionEpoch == -1) { + rd_kafka_mock_sgrp_release_session_locks(sgrp, session); + TAILQ_REMOVE(&sgrp->fetch_sessions, session, link); + sgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + session = NULL; + } + + mtx_unlock(&mcluster->lock); + + /* Emulate real broker MaxWaitMs behaviour: when no + * records were acquired, delay the response to avoid + * a tight fetch loop that starves the client's + * shutdown path. Cap the delay at 100ms so tests + * remain fast. */ + if (acquired_cnt == 0 && MaxWaitMs > 0) { + int32_t delay_ms = MaxWaitMs < 100 ? MaxWaitMs : 100; + resp->rkbuf_ts_sent = (rd_ts_t)delay_ms * 1000; /* us */ + } + + rd_kafka_mock_connection_send_response0(mconn, resp, rd_true); + + rd_kafka_topic_partition_list_destroy(requested_partitions); + RD_IF_FREE(forgotten_partitions, + rd_kafka_topic_partition_list_destroy); + rd_list_destroy(&ack_entries); + return 0; + } + + /* Error response */ + rd_kafka_buf_write_i32(resp, 0); + rd_kafka_buf_write_i16(resp, err); + rd_kafka_buf_write_str(resp, rd_kafka_err2str(err), -1); + rd_kafka_buf_write_i32(resp, 0); + rd_kafka_buf_write_arraycnt(resp, 0); + rd_kafka_buf_write_arraycnt(resp, 0); + rd_kafka_buf_write_tags_empty(resp); + + rd_kafka_mock_connection_send_response0(mconn, resp, rd_true); + rd_kafka_topic_partition_list_destroy(requested_partitions); + RD_IF_FREE(forgotten_partitions, rd_kafka_topic_partition_list_destroy); + rd_list_destroy(&ack_entries); + return 0; + +err_parse: + RD_IF_FREE(requested_partitions, rd_kafka_topic_partition_list_destroy); + RD_IF_FREE(forgotten_partitions, rd_kafka_topic_partition_list_destroy); + rd_list_destroy(&ack_entries); + rd_kafka_buf_destroy(resp); + return -1; +} + +/** + * @brief Handle ShareAcknowledgeRequest. + * + * This is the standalone acknowledgement API (key 79). It has the same + * acknowledgement-batch wire format as the piggy-backed acks inside + * ShareFetch, but does NOT return any fetched records. + * + * Request: GroupId, MemberId, ShareSessionEpoch, Topics[TopicId, + * Partitions[PartitionIndex, AcknowledgementBatches[FirstOffset, + * LastOffset, AcknowledgeTypes[]]]] + * + * Response: ThrottleTimeMs, ErrorCode, ErrorMessage, + * Responses[TopicId, Partitions[PartitionIndex, ErrorCode, + * ErrorMessage, CurrentLeader{LeaderId, LeaderEpoch}]], + * NodeEndpoints[] + */ +static int +rd_kafka_mock_handle_ShareAcknowledge(rd_kafka_mock_connection_t *mconn, + rd_kafka_buf_t *rkbuf) { + const rd_bool_t log_decode_errors = rd_true; + rd_kafka_mock_cluster_t *mcluster = mconn->broker->cluster; + rd_kafka_buf_t *resp = rd_kafka_mock_buf_new_response(rkbuf); + rd_kafkap_str_t GroupId, MemberId; + int32_t SessionEpoch = -1; + int32_t TopicsCnt; + rd_kafka_topic_partition_list_t *ack_partitions = NULL; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; + rd_bool_t ack_parse_err = rd_false; + rd_kafka_mock_sharegroup_t *sgrp = NULL; + rd_list_t ack_entries; + rd_kafka_mock_broker_t *node_endpoints[64]; + int node_endpoint_cnt = 0; + int k; + + (void)log_decode_errors; + + rd_list_init(&ack_entries, 8, rd_free); + + /* ---- Parse request ---- */ + + rd_kafka_buf_read_str(rkbuf, &GroupId); + rd_kafka_buf_read_str(rkbuf, &MemberId); + rd_kafka_buf_read_i32(rkbuf, &SessionEpoch); + + ack_partitions = rd_kafka_topic_partition_list_new(0); + + rd_kafka_buf_read_arraycnt(rkbuf, &TopicsCnt, RD_KAFKAP_TOPICS_MAX); + while (TopicsCnt-- > 0) { + rd_kafka_Uuid_t TopicId = RD_KAFKA_UUID_ZERO; + int32_t PartitionCnt; + + rd_kafka_buf_read_uuid(rkbuf, &TopicId); + rd_kafka_buf_read_arraycnt(rkbuf, &PartitionCnt, + RD_KAFKAP_PARTITIONS_MAX); + + while (PartitionCnt-- > 0) { + int32_t Partition; + int32_t AckBatchCnt; + int64_t prev_ack_last = -1; + rd_kafka_topic_partition_t *rktpar; + + rd_kafka_buf_read_i32(rkbuf, &Partition); + rd_kafka_buf_read_arraycnt(rkbuf, &AckBatchCnt, -1); + while (AckBatchCnt-- > 0) { + int32_t AckTypeCnt; + int64_t AckFirstOffset, AckLastOffset; + int64_t range_len, ti; + int8_t *ack_types = NULL; + int8_t single_type = -1; + + rd_kafka_buf_read_i64(rkbuf, &AckFirstOffset); + rd_kafka_buf_read_i64(rkbuf, &AckLastOffset); + + /* Validate ascending order and + * non-overlapping ranges. */ + if (prev_ack_last >= 0 && + AckFirstOffset <= prev_ack_last) + ack_parse_err = rd_true; + prev_ack_last = AckLastOffset; + + range_len = AckLastOffset - AckFirstOffset + 1; + + rd_kafka_buf_read_arraycnt(rkbuf, &AckTypeCnt, + -1); + + if (AckTypeCnt == 1) { + /* Single type for entire range */ + rd_kafka_buf_read_i8(rkbuf, + &single_type); + } else if (AckTypeCnt > 1) { + /* Per-offset types */ + ack_types = + rd_alloca((size_t)AckTypeCnt * + sizeof(*ack_types)); + for (ti = 0; ti < AckTypeCnt; ti++) + rd_kafka_buf_read_i8( + rkbuf, &ack_types[ti]); + } + rd_kafka_buf_skip_tags(rkbuf); + + if (AckTypeCnt == 1 && single_type >= 0) { + rd_list_add( + &ack_entries, + rd_kafka_mock_sgrp_ack_entry_new( + TopicId, Partition, + AckFirstOffset, AckLastOffset, + single_type)); + } else if (ack_types && + AckTypeCnt == range_len) { + for (ti = 0; ti < range_len; ti++) { + rd_list_add( + &ack_entries, + rd_kafka_mock_sgrp_ack_entry_new( + TopicId, Partition, + AckFirstOffset + ti, + AckFirstOffset + ti, + ack_types[ti])); + } + } else if (AckTypeCnt > 0) { + ack_parse_err = rd_true; + } + } + + rktpar = rd_kafka_topic_partition_list_add( + ack_partitions, "", Partition); + rd_kafka_topic_partition_set_topic_id(rktpar, TopicId); + + rd_kafka_buf_skip_tags(rkbuf); + } + + rd_kafka_buf_skip_tags(rkbuf); + } + + /* Top-level tags */ + rd_kafka_buf_skip_tags(rkbuf); + + rd_kafka_dbg(mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareAcknowledge parsed: group %.*s member %.*s " + "session_epoch %" PRId32 " ack_entries %d", + RD_KAFKAP_STR_PR(&GroupId), RD_KAFKAP_STR_PR(&MemberId), + SessionEpoch, rd_list_cnt(&ack_entries)); + + /* ---- Inject errors if configured ---- */ + err = rd_kafka_mock_next_request_error(mconn, resp); + + if (!err && ack_parse_err) + err = RD_KAFKA_RESP_ERR_INVALID_REQUEST; + + if (!err) { + rd_kafka_mock_sgrp_fetch_session_t *session = NULL; + + mtx_lock(&mcluster->lock); + sgrp = rd_kafka_mock_sharegroup_get(mcluster, &GroupId); + + /* Common validation: member check, session lookup, + * epoch -1 close, epoch > 0 validation. */ + err = rd_kafka_mock_sgrp_session_validate( + sgrp, &MemberId, mconn->broker->id, SessionEpoch, &session, + "ShareAcknowledge"); + + /* ShareAcknowledge must not use epoch=0 + * (only ShareFetch can open sessions). */ + if (!err && SessionEpoch == 0) { + err = RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH; + } + + /* For all successful, non-close requests: update activity + * timestamp and increment epoch for next request. + * Skip for epoch=-1 (final/close). */ + if (!err && session && SessionEpoch != -1) { + session->ts_last_activity = rd_clock(); + session->session_epoch++; + } + + /* Pop injected partition-level errors and stash them on + * the request-local partition list. Done before ack + * application so that acks for an errored partition are + * NOT applied, keeping mock record state consistent with + * the reported error. */ + if (!err) { + int p; + for (p = 0; p < ack_partitions->cnt; p++) { + rd_kafka_topic_partition_t *rktpar = + &ack_partitions->elems[p]; + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find_by_id( + mcluster, + rd_kafka_topic_partition_get_topic_id( + rktpar)); + rd_kafka_mock_partition_t *mpart = + mtopic ? rd_kafka_mock_partition_find( + mtopic, rktpar->partition) + : NULL; + + rktpar->err = RD_KAFKA_RESP_ERR_NO_ERROR; + if (mpart && mpart->leader == mconn->broker) + rktpar->err = + rd_kafka_mock_partition_next_request_error( + mpart, + rkbuf->rkbuf_reqhdr.ApiKey); + } + } + + /* Apply acknowledgement batches */ + if (!err && sgrp && rd_list_cnt(&ack_entries) > 0) { + int k; + rd_kafka_dbg( + mconn->broker->cluster->rk, MOCK, "MOCK", + "ShareAcknowledge: applying %d acknowledgement " + "batch(es) for member %.*s", + rd_list_cnt(&ack_entries), + RD_KAFKAP_STR_PR(&MemberId)); + for (k = 0; k < rd_list_cnt(&ack_entries); k++) { + struct rd_kafka_mock_sgrp_ack_entry *entry = + rd_list_elem(&ack_entries, k); + int idx = + rd_kafka_topic_partition_list_find_idx_by_id( + ack_partitions, entry->topic_id, + entry->partition); + + /* Skip ack application for partitions with + * an injected error. */ + if (idx >= 0 && ack_partitions->elems[idx].err) + continue; + + entry->err = rd_kafka_mock_sgrp_apply_ack( + sgrp, entry->topic_id, entry->partition, + entry->first_offset, entry->last_offset, + entry->ack_type, &MemberId); + } + } + + /* epoch=-1 (final ack) → release remaining + * acquired records and close the session. */ + if (!err && session && SessionEpoch == -1) { + rd_kafka_mock_sgrp_release_session_locks(sgrp, session); + TAILQ_REMOVE(&sgrp->fetch_sessions, session, link); + sgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + session = NULL; + } + + /* ---- Write success response ---- */ + + /* ThrottleTimeMs */ + rd_kafka_buf_write_i32(resp, 0); + /* ErrorCode */ + rd_kafka_buf_write_i16(resp, err); + /* ErrorMessage */ + if (err) + rd_kafka_buf_write_str(resp, rd_kafka_err2str(err), -1); + else + rd_kafka_buf_write_str(resp, NULL, -1); + + /* Responses (per-topic) */ + rd_kafka_topic_partition_list_sort_by_topic_id(ack_partitions); + + { + int i = 0; + int topic_cnt = 0; + rd_kafka_Uuid_t current_topic = RD_KAFKA_UUID_ZERO; + + /* Count distinct topics */ + for (i = 0; i < ack_partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = + &ack_partitions->elems[i]; + rd_kafka_Uuid_t topic_id = + rd_kafka_topic_partition_get_topic_id( + rktpar); + if (i == 0 || + rd_kafka_Uuid_cmp(topic_id, + current_topic) != 0) { + topic_cnt++; + current_topic = topic_id; + } + } + + /* #Responses (topics) */ + rd_kafka_buf_write_arraycnt(resp, topic_cnt); + + i = 0; + while (i < ack_partitions->cnt) { + int j; + rd_kafka_Uuid_t topic_id = + rd_kafka_topic_partition_get_topic_id( + &ack_partitions->elems[i]); + int part_cnt = 0; + + /* Count partitions for this topic */ + for (j = i; j < ack_partitions->cnt; j++) { + rd_kafka_Uuid_t next_topic_id = + rd_kafka_topic_partition_get_topic_id( + &ack_partitions->elems[j]); + if (rd_kafka_Uuid_cmp( + topic_id, next_topic_id) != 0) + break; + part_cnt++; + } + + /* TopicId */ + rd_kafka_buf_write_uuid(resp, &topic_id); + /* #Partitions */ + rd_kafka_buf_write_arraycnt(resp, part_cnt); + + for (j = i; j < i + part_cnt; j++) { + rd_kafka_topic_partition_t *rktpar = + &ack_partitions->elems[j]; + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find_by_id( + mcluster, topic_id); + rd_kafka_mock_partition_t *mpart = + mtopic + ? rd_kafka_mock_partition_find( + mtopic, rktpar->partition) + : NULL; + rd_kafka_resp_err_t part_err; + + if (!mpart) + part_err = + RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + else if (mpart->leader != mconn->broker) + part_err = + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER; + else if (rktpar->err) + /* Injected partition error + * stashed before ack + * application. */ + part_err = rktpar->err; + else + part_err = + rd_kafka_mock_sgrp_ack_error_for_partition( + &ack_entries, topic_id, + rktpar->partition); + + /* PartitionIndex */ + rd_kafka_buf_write_i32( + resp, rktpar->partition); + /* ErrorCode */ + rd_kafka_buf_write_i16(resp, part_err); + /* ErrorMessage */ + if (part_err) + rd_kafka_buf_write_str( + resp, + rd_kafka_err2str(part_err), + -1); + else + rd_kafka_buf_write_str( + resp, NULL, -1); + /* CurrentLeader — populated only + * when the per-partition error + * signals stale leader info. */ + if (mpart && mpart->leader && + (part_err == + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER || + part_err == + RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH)) { + int found = 0; + rd_kafka_buf_write_i32( + resp, mpart->leader->id); + rd_kafka_buf_write_i32( + resp, mpart->leader_epoch); + for (k = 0; + k < node_endpoint_cnt; + k++) { + if (node_endpoints[k] == + mpart->leader) { + found = 1; + break; + } + } + if (!found && + node_endpoint_cnt < 64) + node_endpoints + [node_endpoint_cnt++] = + mpart->leader; + } else { + rd_kafka_buf_write_i32( + resp, -1); /* LeaderId */ + rd_kafka_buf_write_i32( + resp, -1); /* LeaderEpoch */ + } + /* CurrentLeader tags */ + rd_kafka_buf_write_tags_empty(resp); + + /* Partition tags */ + rd_kafka_buf_write_tags_empty(resp); + } + + /* Topic tags */ + rd_kafka_buf_write_tags_empty(resp); + + i += part_cnt; + } + } + + /* NodeEndpoints */ + rd_kafka_buf_write_arraycnt(resp, node_endpoint_cnt); + for (k = 0; k < node_endpoint_cnt; k++) { + rd_kafka_mock_broker_t *nb = node_endpoints[k]; + rd_kafka_buf_write_i32(resp, nb->id); + rd_kafka_buf_write_str(resp, nb->advertised_listener, + -1); + rd_kafka_buf_write_i32(resp, (int32_t)nb->port); + rd_kafka_buf_write_str(resp, nb->rack, -1); + rd_kafka_buf_write_tags_empty(resp); + } + /* Top-level tags */ + rd_kafka_buf_write_tags_empty(resp); + + mtx_unlock(&mcluster->lock); + + rd_kafka_mock_connection_send_response0(mconn, resp, rd_true); + + rd_kafka_topic_partition_list_destroy(ack_partitions); + rd_list_destroy(&ack_entries); + return 0; + } + + /* ---- Error response ---- */ + rd_kafka_buf_write_i32(resp, 0); /* ThrottleTimeMs */ + rd_kafka_buf_write_i16(resp, err); /* ErrorCode */ + rd_kafka_buf_write_str(resp, rd_kafka_err2str(err), -1); /* ErrorMsg */ + rd_kafka_buf_write_arraycnt(resp, 0); /* Responses (empty) */ + rd_kafka_buf_write_arraycnt(resp, 0); /* NodeEndpoints (empty) */ + rd_kafka_buf_write_tags_empty(resp); + + rd_kafka_mock_connection_send_response0(mconn, resp, rd_true); + rd_kafka_topic_partition_list_destroy(ack_partitions); + rd_list_destroy(&ack_entries); + return 0; + +err_parse: + RD_IF_FREE(ack_partitions, rd_kafka_topic_partition_list_destroy); + rd_list_destroy(&ack_entries); + rd_kafka_buf_destroy(resp); + return -1; +} + /** * @brief Default request handlers */ @@ -3023,10 +5162,15 @@ const struct rd_kafka_mock_api_handler {2, 2, -1, rd_kafka_mock_handle_OffsetForLeaderEpoch}, [RD_KAFKAP_ConsumerGroupHeartbeat] = {1, 1, 1, rd_kafka_mock_handle_ConsumerGroupHeartbeat}, + [RD_KAFKAP_ShareGroupHeartbeat] = + {1, 1, 1, rd_kafka_mock_handle_ShareGroupHeartbeat}, [RD_KAFKAP_GetTelemetrySubscriptions] = {0, 0, 0, rd_kafka_mock_handle_GetTelemetrySubscriptions}, [RD_KAFKAP_PushTelemetry] = {0, 0, 0, rd_kafka_mock_handle_PushTelemetry}, + [RD_KAFKAP_ShareFetch] = {1, 1, 1, rd_kafka_mock_handle_ShareFetch}, + [RD_KAFKAP_ShareAcknowledge] = {1, 1, 1, + rd_kafka_mock_handle_ShareAcknowledge}, }; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_mock_int.h b/lib/librdkafka-2.15.0/src/rdkafka_mock_int.h similarity index 67% rename from lib/librdkafka-2.10.1/src/rdkafka_mock_int.h rename to lib/librdkafka-2.15.0/src/rdkafka_mock_int.h index 372e8f4e4fe..1fe050cb1cb 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_mock_int.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_mock_int.h @@ -170,12 +170,124 @@ typedef struct rd_kafka_mock_cgrp_consumer_member_s { rd_list_t *subscribed_topic_names; /**< Subscribed topic names received in the heartbeat */ char *subscribed_topic_regex; /**< Subscribed regex */ + + rd_bool_t left_static_membership; /**< Member left the group + * with static membership. */ struct rd_kafka_mock_connection_s *conn; /**< Connection, may be NULL * if there is no ongoing * request. */ rd_kafka_mock_cgrp_consumer_t *mcgrp; /**< Consumer group */ } rd_kafka_mock_cgrp_consumer_member_t; +/** + * @brief Share record state. + */ +enum rd_kafka_mock_sgrp_record_state_e { + RD_KAFKA_MOCK_SGRP_RECORD_AVAILABLE = 0, + RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED = 1, + RD_KAFKA_MOCK_SGRP_RECORD_ACKNOWLEDGED = 2, + RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED = 3 +}; + +typedef struct rd_kafka_mock_sgrp_record_state_s { + TAILQ_ENTRY(rd_kafka_mock_sgrp_record_state_s) link; + int64_t offset; + char *owner_member_id; + rd_ts_t lock_expiry_ts; + int32_t delivery_count; + enum rd_kafka_mock_sgrp_record_state_e state; +} rd_kafka_mock_sgrp_record_state_t; + +/** + * @brief Share partition metadata. + */ +typedef struct rd_kafka_mock_sgrp_partmeta_s { + TAILQ_ENTRY(rd_kafka_mock_sgrp_partmeta_s) link; + rd_kafka_Uuid_t topic_id; + int32_t partition; + int64_t spso; + int64_t speo; + TAILQ_HEAD(, rd_kafka_mock_sgrp_record_state_s) inflight; + int inflight_cnt; + int acquired_cnt; /**< Number of records in ACQUIRED state */ +} rd_kafka_mock_sgrp_partmeta_t; + +/** + * @brief Share fetch session. + */ +typedef struct rd_kafka_mock_sgrp_fetch_session_s { + TAILQ_ENTRY(rd_kafka_mock_sgrp_fetch_session_s) link; + char *member_id; + int32_t node_id; /**< Node ID of the broker owning this session */ + int32_t session_id; + int32_t session_epoch; + rd_ts_t ts_last_activity; + rd_kafka_topic_partition_list_t *partitions; + int partition_start_idx; /**< Rotation index for starvation prevention + */ +} rd_kafka_mock_sgrp_fetch_session_t; + +/** + * @struct Share group. + */ +typedef struct rd_kafka_mock_sharegroup_s { + TAILQ_ENTRY(rd_kafka_mock_sharegroup_s) link; + struct rd_kafka_mock_cluster_s *cluster; /**< Cluster */ + char *id; /**< Share group Id */ + int32_t group_epoch; /**< Group epoch */ + int session_timeout_ms; /**< Session timeout */ + rd_kafka_timer_t session_tmr; /**< Session timeout timer */ + int heartbeat_interval_ms; /**< Heartbeat interval */ + TAILQ_HEAD(, rd_kafka_mock_sharegroup_member_s) + members; /**< Share group members */ + int member_cnt; /**< Number of share group members */ + rd_bool_t manual_assignment; /**< Use manual assignment */ + + /* ShareFetch state */ + TAILQ_HEAD(, rd_kafka_mock_sgrp_partmeta_s) + partitions; /**< Share partition metadata */ + int partition_cnt; /**< Number of partitions */ + TAILQ_HEAD(, rd_kafka_mock_sgrp_fetch_session_s) + fetch_sessions; /**< Active fetch sessions */ + int fetch_session_cnt; /**< Number of fetch sessions */ + rd_kafka_timer_t fetch_session_tmr; /**< Fetch session expiry timer */ + + /* Per-record limits */ + int max_delivery_attempts; /**< Max times a record can be + * acquired before being archived. + * 0 = unlimited (default 5). */ + int record_lock_duration_ms; /**< Per-record lock duration in ms. + * 0 = use session_timeout_ms + * as fallback (default 0). */ + int isolation_level; /**< Share isolation level */ + int max_size; /**< Max members allowed. + * 0 = unlimited (default). */ + int max_fetch_sessions; /**< Max fetch sessions per broker. + * 0 = unlimited (default 2000). */ + int max_record_locks; /**< Max acquired record locks per + * share-partition. + * 0 = unlimited (default 2000). */ + int auto_offset_reset; /**< 0 = latest (default), + * 1 = earliest. */ +} rd_kafka_mock_sharegroup_t; + +/** + * @struct Share group member. + */ +typedef struct rd_kafka_mock_sharegroup_member_s { + TAILQ_ENTRY(rd_kafka_mock_sharegroup_member_s) link; + char *id; /**< MemberId */ + rd_ts_t ts_last_activity; /**< Last heartbeat timestamp */ + int32_t member_epoch; /**< Current member epoch */ + int32_t previous_member_epoch; /**< Previous member epoch (allows + * client to catch up if response + * with bumped epoch was lost) */ + rd_list_t *subscribed_topic_names; /**< Subscribed topic names */ + rd_kafka_topic_partition_list_t *assignment; /**< Current assignment */ + struct rd_kafka_mock_connection_s *conn; /**< Connection */ + rd_kafka_mock_sharegroup_t *mshgrp; /**< Share group */ +} rd_kafka_mock_sharegroup_member_t; + /** * @struct TransactionalId + PID (+ optional sequence state) @@ -192,6 +304,56 @@ typedef struct rd_kafka_mock_pid_s { char TransactionalId[1]; /**< Allocated after this structure */ } rd_kafka_mock_pid_t; +/** + * @brief Transaction state enum. + */ +typedef enum { + RD_KAFKA_MOCK_TXN_NONE, /**< No active transaction */ + RD_KAFKA_MOCK_TXN_ONGOING, /**< Transaction in progress */ +} rd_kafka_mock_txn_state_t; + +/** + * @struct A partition that is part of an active transaction. + * + * Linked from rd_kafka_mock_txn_t.partitions. + */ +typedef struct rd_kafka_mock_txn_partition_s { + TAILQ_ENTRY(rd_kafka_mock_txn_partition_s) link; + char *topic_name; + int32_t partition; + int64_t first_offset; /**< First offset produced in this txn, + * -1 if no data produced yet. */ +} rd_kafka_mock_txn_partition_t; + +/** + * @brief Active transaction state for a transactional producer. + * + * One per TransactionalId, stored in mcluster->transactions. + */ +typedef struct rd_kafka_mock_txn_s { + TAILQ_ENTRY(rd_kafka_mock_txn_s) link; + char *transactional_id; + rd_kafka_pid_t pid; /**< Current PID for this txn */ + rd_kafka_mock_txn_state_t state; + + /** Partitions registered via AddPartitionsToTxn */ + TAILQ_HEAD(, rd_kafka_mock_txn_partition_s) partitions; + int partition_cnt; +} rd_kafka_mock_txn_t; + +/** + * @brief A completed (aborted) transaction range on a partition. + * + * Stored in rd_kafka_mock_partition_t.aborted_txns. + * Used during read_committed ShareFetch to filter out aborted data. + */ +typedef struct rd_kafka_mock_aborted_txn_s { + TAILQ_ENTRY(rd_kafka_mock_aborted_txn_s) link; + int64_t pid_id; /**< Producer ID */ + int64_t first_offset; /**< First offset of aborted txn on this part */ + int64_t last_offset; /**< Last data offset (before control record) */ +} rd_kafka_mock_aborted_txn_t; + /** * @brief rd_kafka_mock_pid_t.pid Pid (not epoch) comparator */ @@ -259,6 +421,13 @@ typedef struct rd_kafka_mock_broker_s { uint16_t port; char *rack; rd_bool_t up; + /**< If false, broker is hidden from Metadata responses and skipped + * when assigning partition replicas, but its TCP connections and + * listen socket remain active so in-flight requests can still + * complete. Used by rd_kafka_mock_broker_remove_from_metadata to + * simulate a KIP-932 broker decommission seen via metadata + * refresh, decoupled from the connection drop. Default: rd_true. */ + rd_bool_t in_metadata; rd_ts_t rtt; /**< RTT in microseconds */ rd_socket_t listen_s; /**< listen() socket */ @@ -345,6 +514,13 @@ typedef struct rd_kafka_mock_partition_s { rd_list_t pidstates; /**< PID states */ + /** Aborted transaction ranges for read_committed filtering */ + TAILQ_HEAD(, rd_kafka_mock_aborted_txn_s) aborted_txns; + + /** Last Stable Offset: offset of first open transaction. + * If no open transactions, lso == end_offset. */ + int64_t lso; + int32_t follower_id; /**< Preferred replica/follower */ struct rd_kafka_mock_topic_s *topic; @@ -352,6 +528,12 @@ typedef struct rd_kafka_mock_partition_s { /**< Leader responses */ TAILQ_HEAD(, rd_kafka_mock_partition_leader_s) leader_responses; + + /**< Per-ApiKey request error stack. + * Pushed via RD_KAFKA_MOCK_CMD_PART_PUSH_REQUEST_ERRORS and + * popped by the protocol request handlers. + * Only accessed from the mock cluster thread, no locks needed. */ + rd_kafka_mock_error_stack_head_t errstacks; } rd_kafka_mock_partition_t; @@ -426,6 +608,8 @@ struct rd_kafka_mock_cluster_s { TAILQ_HEAD(, rd_kafka_mock_cgrp_consumer_s) cgrps_consumer; + TAILQ_HEAD(, rd_kafka_mock_sharegroup_s) sharegrps; + /** Explicit coordinators (set with mock_set_coordinator()) */ TAILQ_HEAD(, rd_kafka_mock_coord_s) coords; @@ -433,6 +617,9 @@ struct rd_kafka_mock_cluster_s { * Element type is a malloced rd_kafka_mock_pid_t*. */ rd_list_t pids; + /** Active transactions by TransactionalId. */ + TAILQ_HEAD(, rd_kafka_mock_txn_s) transactions; + char *bootstraps; /**< bootstrap.servers */ thrd_t thread; /**< Mock thread */ @@ -464,6 +651,31 @@ struct rd_kafka_mock_cluster_s { int group_consumer_session_timeout_ms; /** Heartbeat interval (KIP 848) */ int group_consumer_heartbeat_interval_ms; + /** Session timeout */ + int sharegroup_session_timeout_ms; + /** Heartbeat interval */ + int sharegroup_heartbeat_interval_ms; + /** Max delivery attempts per record. + * 0 = unlimited. */ + int sharegroup_max_delivery_attempts; + /** Per-record lock duration in ms. + * 0 = use session_timeout_ms. */ + int sharegroup_record_lock_duration_ms; + /** Share isolation level. + * 0 = read_uncommitted, 1 = read_committed. */ + int sharegroup_isolation_level; + /** Max members allowed in share group. + * 0 = unlimited. */ + int sharegroup_max_size; + /** Max fetch sessions per broker. + * 0 = unlimited. */ + int sharegroup_max_fetch_sessions; + /** Max acquired record locks per share-partition. + * 0 = unlimited. */ + int sharegroup_max_record_locks; + /** Auto offset reset. + * 0 = latest (default), 1 = earliest. */ + int sharegroup_auto_offset_reset; } defaults; /**< Dynamic array of IO handlers for corresponding fd in .fds */ @@ -538,6 +750,25 @@ rd_kafka_mock_topic_t * rd_kafka_mock_topic_find_by_id(const rd_kafka_mock_cluster_t *mcluster, rd_kafka_Uuid_t id); +void rd_kafka_mock_sgrp_fetch_session_destroy( + rd_kafka_mock_sgrp_fetch_session_t *session); +rd_kafka_resp_err_t rd_kafka_mock_sgrp_session_validate( + rd_kafka_mock_sharegroup_t *sgrp, + const rd_kafkap_str_t *MemberId, + int32_t NodeId, + int32_t SessionEpoch, + rd_kafka_mock_sgrp_fetch_session_t **sessionp, + const char *api_name); +void rd_kafka_mock_sgrp_record_release( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sgrp_partmeta_t *pmeta, + rd_kafka_mock_sgrp_record_state_t *state); +void rd_kafka_mock_sgrp_release_session_locks( + rd_kafka_mock_sharegroup_t *mshgrp, + const rd_kafka_mock_sgrp_fetch_session_t *session); +void rd_kafka_mock_sgrp_fetch_session_tmr_cb(rd_kafka_timers_t *rkts, + void *arg); + rd_kafka_mock_broker_t * rd_kafka_mock_cluster_get_coord(rd_kafka_mock_cluster_t *mcluster, rd_kafka_coordtype_t KeyType, @@ -561,6 +792,10 @@ rd_kafka_resp_err_t rd_kafka_mock_next_request_error(rd_kafka_mock_connection_t *mconn, rd_kafka_buf_t *resp); +rd_kafka_resp_err_t +rd_kafka_mock_partition_next_request_error(rd_kafka_mock_partition_t *mpart, + int16_t ApiKey); + rd_kafka_resp_err_t rd_kafka_mock_partition_log_append(rd_kafka_mock_partition_t *mpart, const rd_kafkap_bytes_t *records, @@ -571,6 +806,21 @@ rd_kafka_resp_err_t rd_kafka_mock_partition_leader_epoch_check( const rd_kafka_mock_partition_t *mpart, int32_t leader_epoch); +/* Transaction helpers */ +rd_kafka_mock_txn_t * +rd_kafka_mock_txn_get(rd_kafka_mock_cluster_t *mcluster, + const rd_kafkap_str_t *TransactionalId); +void rd_kafka_mock_txn_partition_add(rd_kafka_mock_txn_t *mtxn, + const char *topic, + int32_t partition); +void rd_kafka_mock_txn_partitions_clear(rd_kafka_mock_txn_t *mtxn); +void rd_kafka_mock_partition_update_lso(rd_kafka_mock_partition_t *mpart, + rd_kafka_mock_cluster_t *mcluster); +void rd_kafka_mock_partition_write_control_batch( + rd_kafka_mock_partition_t *mpart, + rd_kafka_pid_t pid, + rd_bool_t committed); + int64_t rd_kafka_mock_partition_offset_for_leader_epoch( const rd_kafka_mock_partition_t *mpart, int32_t leader_epoch); @@ -675,7 +925,8 @@ rd_kafka_mock_cgrp_consumer_get(rd_kafka_mock_cluster_t *mcluster, void rd_kafka_mock_cgrp_consumer_member_leave( rd_kafka_mock_cgrp_consumer_t *mcgrp, - rd_kafka_mock_cgrp_consumer_member_t *member); + rd_kafka_mock_cgrp_consumer_member_t *member, + rd_bool_t static_leave); void rd_kafka_mock_cgrp_consumer_member_fenced( rd_kafka_mock_cgrp_consumer_t *mcgrp, @@ -696,6 +947,54 @@ rd_kafka_mock_cgrp_consumer_member_t *rd_kafka_mock_cgrp_consumer_member_add( void rd_kafka_mock_cgrps_connection_closed(rd_kafka_mock_cluster_t *mcluster, rd_kafka_mock_connection_t *mconn); + +/* Share group */ + +void rd_kafka_mock_sharegrps_init(rd_kafka_mock_cluster_t *mcluster); + +rd_kafka_mock_sharegroup_t * +rd_kafka_mock_sharegroup_find(rd_kafka_mock_cluster_t *mcluster, + const rd_kafkap_str_t *GroupId); + +rd_kafka_mock_sharegroup_t * +rd_kafka_mock_sharegroup_get(rd_kafka_mock_cluster_t *mcluster, + const rd_kafkap_str_t *GroupId); + +void rd_kafka_mock_sharegroup_destroy(rd_kafka_mock_sharegroup_t *mshgrp); + +rd_kafka_mock_sharegroup_member_t * +rd_kafka_mock_sharegroup_member_find(rd_kafka_mock_sharegroup_t *mshgrp, + const rd_kafkap_str_t *MemberId); + +void rd_kafka_mock_sharegroup_member_destroy( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sharegroup_member_t *member); + +void rd_kafka_mock_sharegroup_member_active( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sharegroup_member_t *member); + +rd_kafka_mock_sharegroup_member_t * +rd_kafka_mock_sharegroup_member_get(rd_kafka_mock_sharegroup_t *mshgrp, + const rd_kafkap_str_t *MemberID, + int32_t MemberEpoch, + rd_kafka_mock_connection_t *mconn); + +rd_bool_t rd_kafka_mock_sharegroup_member_subscribed_topic_names_set( + rd_kafka_mock_sharegroup_member_t *member, + const rd_kafkap_str_t *SubscribedTopicNames, + int32_t SubscribedTopicNamesCnt); + +void rd_kafka_mock_sharegroup_assignment_recalculate( + rd_kafka_mock_sharegroup_t *mshgrp); + +void rd_kafka_mock_sharegrps_connection_closed( + rd_kafka_mock_cluster_t *mcluster, + rd_kafka_mock_connection_t *mconn); +void rd_kafka_mock_sharegrps_node_connection_closed( + rd_kafka_mock_cluster_t *mcluster, + int32_t node_id); + /** *@} */ diff --git a/lib/librdkafka-2.15.0/src/rdkafka_mock_sharegrp.c b/lib/librdkafka-2.15.0/src/rdkafka_mock_sharegrp.c new file mode 100644 index 00000000000..7c3b7592e4c --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdkafka_mock_sharegrp.c @@ -0,0 +1,1207 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Mocks + * + */ + + +#include "rdkafka_int.h" +#include "rdbuf.h" +#include "rdkafka_mock_int.h" +#include + +/** + * @brief Share group target assignment (manual) + */ +typedef struct rd_kafka_mock_sharegroup_target_assignments_s { + rd_list_t member_ids; /**< List of member ids (char *) */ + rd_list_t assignment; /**< List of rd_kafka_topic_partition_list_t */ +} rd_kafka_mock_sharegroup_target_assignment_t; + +/* Forward declarations */ +static void rd_kafka_mock_sharegroup_session_tmr_cb(rd_kafka_timers_t *rkts, + void *arg); + +/** + * @brief Initializes sharegroups in mock cluster + */ +void rd_kafka_mock_sharegrps_init(rd_kafka_mock_cluster_t *mcluster) { + TAILQ_INIT(&mcluster->sharegrps); + mcluster->defaults.sharegroup_session_timeout_ms = 45000; + mcluster->defaults.sharegroup_heartbeat_interval_ms = 5000; + mcluster->defaults.sharegroup_max_delivery_attempts = 5; + mcluster->defaults.sharegroup_record_lock_duration_ms = 30000; + mcluster->defaults.sharegroup_max_size = 200; + mcluster->defaults.sharegroup_isolation_level = 0; + mcluster->defaults.sharegroup_max_fetch_sessions = 2000; + mcluster->defaults.sharegroup_max_record_locks = 2000; + mcluster->defaults.sharegroup_auto_offset_reset = 0; /* latest */ +} + +/** + * @brief Find a share group by GroupId. + */ +rd_kafka_mock_sharegroup_t * +rd_kafka_mock_sharegroup_find(rd_kafka_mock_cluster_t *mcluster, + const rd_kafkap_str_t *GroupId) { + rd_kafka_mock_sharegroup_t *mshgrp; + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) { + if (!rd_kafkap_str_cmp_str(GroupId, mshgrp->id)) + return mshgrp; + } + return NULL; +} + +/** + * @brief Get or create a share group + */ +rd_kafka_mock_sharegroup_t * +rd_kafka_mock_sharegroup_get(rd_kafka_mock_cluster_t *mcluster, + const rd_kafkap_str_t *GroupID) { + rd_kafka_mock_sharegroup_t *mshgrp; + + /* Check if the share group already exists */ + mshgrp = rd_kafka_mock_sharegroup_find(mcluster, GroupID); + if (mshgrp) + return mshgrp; + + /* Create new share group */ + mshgrp = rd_calloc(1, sizeof(*mshgrp)); + mshgrp->cluster = mcluster; + mshgrp->id = RD_KAFKAP_STR_DUP(GroupID); + mshgrp->group_epoch = 1; + mshgrp->session_timeout_ms = + mcluster->defaults.sharegroup_session_timeout_ms; + mshgrp->heartbeat_interval_ms = + mcluster->defaults.sharegroup_heartbeat_interval_ms; + + TAILQ_INIT(&mshgrp->members); + mshgrp->member_cnt = 0; + + /* ShareFetch state */ + TAILQ_INIT(&mshgrp->partitions); + TAILQ_INIT(&mshgrp->fetch_sessions); + mshgrp->partition_cnt = 0; + mshgrp->fetch_session_cnt = 0; + + /* Per-record limits */ + mshgrp->max_delivery_attempts = + mcluster->defaults.sharegroup_max_delivery_attempts; + mshgrp->record_lock_duration_ms = + mcluster->defaults.sharegroup_record_lock_duration_ms; + mshgrp->isolation_level = mcluster->defaults.sharegroup_isolation_level; + mshgrp->max_size = mcluster->defaults.sharegroup_max_size; + mshgrp->max_fetch_sessions = + mcluster->defaults.sharegroup_max_fetch_sessions; + mshgrp->max_record_locks = + mcluster->defaults.sharegroup_max_record_locks; + mshgrp->auto_offset_reset = + mcluster->defaults.sharegroup_auto_offset_reset; + + rd_kafka_timer_start(&mcluster->timers, &mshgrp->session_tmr, + 1000 * 1000 /* 1s */, + rd_kafka_mock_sharegroup_session_tmr_cb, mshgrp); + + /* Fetch session expiry timer */ + rd_kafka_timer_start(&mcluster->timers, &mshgrp->fetch_session_tmr, + 1000 * 1000 /* 1s */, + rd_kafka_mock_sgrp_fetch_session_tmr_cb, mshgrp); + + TAILQ_INSERT_TAIL(&mcluster->sharegrps, mshgrp, link); + + return mshgrp; +} + +/** + * @brief Destroy a share group + */ +void rd_kafka_mock_sharegroup_destroy(rd_kafka_mock_sharegroup_t *mshgrp) { + rd_kafka_mock_sharegroup_member_t *member; + rd_kafka_mock_sgrp_partmeta_t *pmeta; + rd_kafka_mock_sgrp_fetch_session_t *session; + + TAILQ_REMOVE(&mshgrp->cluster->sharegrps, mshgrp, link); + rd_kafka_timer_stop(&mshgrp->cluster->timers, &mshgrp->session_tmr, + RD_DO_LOCK); + rd_kafka_timer_stop(&mshgrp->cluster->timers, + &mshgrp->fetch_session_tmr, RD_DO_LOCK); + + /* Destroy all members */ + while ((member = TAILQ_FIRST(&mshgrp->members))) + rd_kafka_mock_sharegroup_member_destroy(mshgrp, member); + + /* Destroy ShareFetch partition metadata */ + while ((pmeta = TAILQ_FIRST(&mshgrp->partitions))) { + rd_kafka_mock_sgrp_record_state_t *state; + TAILQ_REMOVE(&mshgrp->partitions, pmeta, link); + while ((state = TAILQ_FIRST(&pmeta->inflight))) { + TAILQ_REMOVE(&pmeta->inflight, state, link); + rd_free(state->owner_member_id); + rd_free(state); + } + rd_free(pmeta); + } + + /* Destroy ShareFetch sessions */ + while ((session = TAILQ_FIRST(&mshgrp->fetch_sessions))) { + TAILQ_REMOVE(&mshgrp->fetch_sessions, session, link); + mshgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + } + + rd_free(mshgrp->id); + rd_free(mshgrp); +} + +/** + * @brief Find a share group member by MemberId. + */ +rd_kafka_mock_sharegroup_member_t * +rd_kafka_mock_sharegroup_member_find(rd_kafka_mock_sharegroup_t *mshgrp, + const rd_kafkap_str_t *MemberId) { + rd_kafka_mock_sharegroup_member_t *member; + TAILQ_FOREACH(member, &mshgrp->members, link) { + if (!rd_kafkap_str_cmp_str(MemberId, member->id)) + return member; + } + return NULL; +} + +/** + * @brief Destroy a share group member. + */ +void rd_kafka_mock_sharegroup_member_destroy( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sharegroup_member_t *member) { + rd_assert(mshgrp->member_cnt > 0); + TAILQ_REMOVE(&mshgrp->members, member, link); + mshgrp->member_cnt--; + rd_free(member->id); + + RD_IF_FREE(member->subscribed_topic_names, rd_list_destroy_free); + RD_IF_FREE(member->assignment, rd_kafka_topic_partition_list_destroy); + rd_free(member); +} + +/** + * @brief Mark member as active. + */ +void rd_kafka_mock_sharegroup_member_active( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sharegroup_member_t *member) { + rd_kafka_dbg(mshgrp->cluster->rk, MOCK, "MOCK", + "Marking mock share group member %s as active", + member->id); + member->ts_last_activity = rd_clock(); +} + +/** + * @brief Fence a member. + */ +void rd_kafka_mock_sharegroup_member_fenced( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sharegroup_member_t *member) { + rd_kafka_dbg(mshgrp->cluster->rk, MOCK, "MOCK", + "Member %s is fenced from sharegroup %s", member->id, + mshgrp->id); + + rd_kafka_mock_sharegroup_member_destroy(mshgrp, member); + + /* Recalculate assignments so remaining members get the + * freed partitions. */ + rd_kafka_mock_sharegroup_assignment_recalculate(mshgrp); +} + +/** + * @brief Check all members for inactivity and remove them if timed out. + */ +static void rd_kafka_mock_sharegroup_session_tmr_cb(rd_kafka_timers_t *rkts, + void *arg) { + rd_kafka_mock_sharegroup_t *mshgrp = arg; + rd_kafka_mock_sharegroup_member_t *member, *tmp; + rd_ts_t now = rd_clock(); + rd_kafka_mock_cluster_t *mcluster = mshgrp->cluster; + + mtx_lock(&mcluster->lock); + TAILQ_FOREACH_SAFE(member, &mshgrp->members, link, tmp) { + if (member->ts_last_activity + + (mshgrp->session_timeout_ms * 1000) > + now) + continue; + + rd_kafka_dbg(mcluster->rk, MOCK, "MOCK", + "Member %s session timed out for sharegroup %s", + member->id, mshgrp->id); + + rd_kafka_mock_sharegroup_member_fenced(mshgrp, member); + } + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Get or create a share group member. + */ +rd_kafka_mock_sharegroup_member_t * +rd_kafka_mock_sharegroup_member_get(rd_kafka_mock_sharegroup_t *mshgrp, + const rd_kafkap_str_t *MemberID, + int32_t MemberEpoch, + rd_kafka_mock_connection_t *mconn) { + rd_kafka_mock_sharegroup_member_t *member; + + /* Check if the member already exists */ + member = rd_kafka_mock_sharegroup_member_find(mshgrp, MemberID); + if (member) { + member->conn = mconn; + rd_kafka_mock_sharegroup_member_active(mshgrp, member); + return member; + } + + /* Only create if epoch is 0 */ + if (MemberEpoch != 0) + return NULL; + + /* Create new member */ + member = rd_calloc(1, sizeof(*member)); + member->mshgrp = mshgrp; + member->id = RD_KAFKAP_STR_DUP(MemberID); + member->member_epoch = mshgrp->group_epoch; + member->previous_member_epoch = + -1; /* No previous epoch for new members */ + member->conn = mconn; + + TAILQ_INSERT_TAIL(&mshgrp->members, member, link); + mshgrp->member_cnt++; + rd_kafka_mock_sharegroup_member_active(mshgrp, member); + + return member; +} + +/** + * @brief Update share group member's subscribed topic names. + * + * @param member The member to update. + * @param SubscribedTopicNames Array of topic names. + * @param SubscribedTopicNamesCnt Count of topic names: + * -1 = unchanged (no modification) + * 0 = clear all subscriptions + * >0 = set to provided topics + * + * @returns rd_true if subscriptions changed, rd_false otherwise. + */ +rd_bool_t rd_kafka_mock_sharegroup_member_subscribed_topic_names_set( + rd_kafka_mock_sharegroup_member_t *member, + const rd_kafkap_str_t *SubscribedTopicNames, + int32_t SubscribedTopicNamesCnt) { + int32_t i; + + if (SubscribedTopicNamesCnt < 0) { + /* -1 means unchanged */ + return rd_false; + } + + if (SubscribedTopicNamesCnt == 0) { + /* 0 means clear all subscriptions */ + if (!member->subscribed_topic_names || + rd_list_cnt(member->subscribed_topic_names) == 0) { + /* Already empty, no change */ + return rd_false; + } + rd_list_destroy(member->subscribed_topic_names); + member->subscribed_topic_names = NULL; + return rd_true; + } + + /* SubscribedTopicNamesCnt > 0: Check if subscription changed */ + if (member->subscribed_topic_names) { + if (rd_list_cnt(member->subscribed_topic_names) == + SubscribedTopicNamesCnt) { + rd_bool_t same = rd_true; + char *topic; + int j; + + RD_LIST_FOREACH(topic, member->subscribed_topic_names, + j) { + rd_bool_t found = rd_false; + for (i = 0; i < SubscribedTopicNamesCnt; i++) { + if (!rd_kafkap_str_cmp_str( + &SubscribedTopicNames[i], + topic)) { + found = rd_true; + break; + } + } + if (!found) { + same = rd_false; + break; + } + } + if (same) + return rd_false; + } + } + + /* Subscription changed, update the list */ + RD_IF_FREE(member->subscribed_topic_names, rd_list_destroy); + member->subscribed_topic_names = + rd_list_new(SubscribedTopicNamesCnt, rd_free); + + for (i = 0; i < SubscribedTopicNamesCnt; i++) { + rd_list_add(member->subscribed_topic_names, + RD_KAFKAP_STR_DUP(&SubscribedTopicNames[i])); + } + + return rd_true; +} + +/** + * @brief Collect all subscribed topic names from all members. + */ +static rd_list_t *rd_kafka_mock_sharegroup_collect_subscribed_topics( + rd_kafka_mock_sharegroup_t *mshgrp) { + rd_kafka_mock_sharegroup_member_t *member; + rd_list_t *all_topics; + + all_topics = rd_list_new(32, rd_free); + + TAILQ_FOREACH(member, &mshgrp->members, link) { + const char *topic; + int i; + + if (!member->subscribed_topic_names) + continue; + + RD_LIST_FOREACH(topic, member->subscribed_topic_names, i) { + const char *existing; + int j; + rd_bool_t found = rd_false; + + /* Check if topic already in all_topics */ + RD_LIST_FOREACH(existing, all_topics, j) { + if (!strcmp(topic, existing)) { + found = rd_true; + break; + } + } + + /* Add if not found */ + if (!found) { + rd_list_add(all_topics, rd_strdup(topic)); + } + } + } + + return all_topics; +} + +/** + * @brief Get list of member ID's subscribed to a topic. + */ +rd_list_t *rd_kafka_mock_sharegroup_get_members_for_topic( + rd_kafka_mock_sharegroup_t *mshgrp, + char *topic_name) { + rd_kafka_mock_sharegroup_member_t *member; + rd_list_t *subscribed_members; + int member_idx = 0; + + subscribed_members = rd_list_new(mshgrp->member_cnt, rd_free); + + TAILQ_FOREACH(member, &mshgrp->members, link) { + char *topic; + int i; + + if (member->subscribed_topic_names) { + RD_LIST_FOREACH(topic, member->subscribed_topic_names, + i) { + if (!strcmp(topic, topic_name)) { + int *idx = rd_malloc(sizeof(*idx)); + *idx = member_idx; + rd_list_add(subscribed_members, idx); + break; + } + } + } + member_idx++; + } + + return subscribed_members; +} + +/** + * @brief Assign partitions of a single topic to subscribed members. + */ +void rd_kafka_mock_sharegroup_assign_topic_partitions( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_topic_t *mtopic, + rd_list_t *subscribed_member_indices) { + int member_count; + int partition_cnt; + int partition_idx; + double precise; + int i; + + member_count = rd_list_cnt(subscribed_member_indices); + partition_cnt = mtopic->partition_cnt; + + if (member_count == 0 || partition_cnt == 0) + return; + + precise = (double)partition_cnt / (double)member_count; + partition_idx = 0; + + for (i = 0; i < member_count; i++) { + int *member_idx_ptr = + (int *)rd_list_elem(subscribed_member_indices, i); + rd_kafka_mock_sharegroup_member_t *member; + int j, cnt = 0; + int num_partitions; + + TAILQ_FOREACH(member, &mshgrp->members, link) { + if (cnt == *member_idx_ptr) + break; + cnt++; + } + + if (!member) + continue; + + num_partitions = (int)ceil(precise * (double)(i + 1)) - + (int)ceil(precise * (double)i); + + if (!member->assignment) + member->assignment = + rd_kafka_topic_partition_list_new(num_partitions); + + for (j = 0; j < num_partitions && partition_idx < partition_cnt; + j++, partition_idx++) { + rd_kafka_topic_partition_t *rktpar; + rktpar = rd_kafka_topic_partition_list_add( + member->assignment, mtopic->name, partition_idx); + /* Set topic ID so the response can include it */ + rd_kafka_topic_partition_set_topic_id(rktpar, + mtopic->id); + } + } +} + +/** + * @brief Recalculate assignments for all members in the share group. + */ +void rd_kafka_mock_sharegroup_assignment_recalculate( + rd_kafka_mock_sharegroup_t *mshgrp) { + rd_kafka_mock_sharegroup_member_t *member; + rd_list_t *all_topics; + char *topic_name; + int i; + + if (mshgrp->member_cnt == 0) + return; + + /* Skip automatic assignment if manual mode is enabled */ + if (mshgrp->manual_assignment) + return; + + TAILQ_FOREACH(member, &mshgrp->members, link) { + if (member->assignment) { + rd_kafka_topic_partition_list_destroy( + member->assignment); + member->assignment = NULL; + } + } + + all_topics = rd_kafka_mock_sharegroup_collect_subscribed_topics(mshgrp); + + RD_LIST_FOREACH(topic_name, all_topics, i) { + rd_kafka_mock_topic_t *mtopic; + rd_list_t *subscribed_members; + + mtopic = rd_kafka_mock_topic_find(mshgrp->cluster, topic_name); + if (!mtopic) + continue; + + subscribed_members = + rd_kafka_mock_sharegroup_get_members_for_topic(mshgrp, + topic_name); + + rd_kafka_mock_sharegroup_assign_topic_partitions( + mshgrp, mtopic, subscribed_members); + + rd_list_destroy(subscribed_members); + } + + mshgrp->group_epoch++; + + TAILQ_FOREACH(member, &mshgrp->members, link) { + /* Save the current epoch as previous before bumping. + * This allows the client to catch up if the response + * with the new epoch was lost. */ + member->previous_member_epoch = member->member_epoch; + member->member_epoch = mshgrp->group_epoch; + } + + rd_list_destroy(all_topics); +} + +/** + * @brief Create a new target assignment (manual) + */ +rd_kafka_mock_sharegroup_target_assignment_t * +rd_kafka_mock_sharegroup_target_assignment_new(void) { + rd_kafka_mock_sharegroup_target_assignment_t *target_assignment; + target_assignment = rd_calloc(1, sizeof(*target_assignment)); + rd_list_init(&target_assignment->member_ids, 0, rd_free); + rd_list_init(&target_assignment->assignment, 0, + (void *)rd_kafka_topic_partition_list_destroy); + + return target_assignment; +} + +/** + * @brief Destroy target assignment + */ +void rd_kafka_mock_sharegroup_target_assignment_destroy( + rd_kafka_mock_sharegroup_target_assignment_t *target_assignment) { + rd_list_destroy(&target_assignment->member_ids); + rd_list_destroy(&target_assignment->assignment); + rd_free(target_assignment); +} + +/** + * @brief Set the target assignment for the sharegroup. + * This applies the manual assignment to the members. + * + * @locks mcluster->lock MUST be held. + */ +static void rd_kafka_mock_sharegroup_target_assignment_set( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sharegroup_target_assignment_t *target_assignment) { + rd_kafka_mock_sharegroup_member_t *member; + int i; + + for (i = 0; i < rd_list_cnt(&target_assignment->member_ids); i++) { + const char *member_id = + rd_list_elem(&target_assignment->member_ids, i); + const rd_kafka_topic_partition_list_t *partitions = + rd_list_elem(&target_assignment->assignment, i); + rd_kafkap_str_t *member_id_str; + + member_id_str = rd_kafkap_str_new(member_id, -1); + member = + rd_kafka_mock_sharegroup_member_find(mshgrp, member_id_str); + rd_kafkap_str_destroy(member_id_str); + + if (!member) { + rd_kafka_dbg(mshgrp->cluster->rk, MOCK, "MOCK", + "Cannot set target assignment for " + "non-existing member %s in sharegroup %s", + member_id, mshgrp->id); + continue; + } + + if (member->assignment) { + rd_kafka_topic_partition_list_destroy( + member->assignment); + } + + member->assignment = + rd_kafka_topic_partition_list_copy(partitions); + + /* Set topic IDs on each partition so the heartbeat response + * can include them (ShareGroupHeartbeat uses topic IDs) */ + { + int j; + for (j = 0; j < member->assignment->cnt; j++) { + rd_kafka_topic_partition_t *rktpar = + &member->assignment->elems[j]; + rd_kafkap_str_t topic_str = { + .str = rktpar->topic, + .len = strlen(rktpar->topic)}; + rd_kafka_mock_topic_t *mtopic = + rd_kafka_mock_topic_find_by_kstr( + mshgrp->cluster, &topic_str); + if (mtopic) { + rd_kafka_topic_partition_set_topic_id( + rktpar, mtopic->id); + } + } + } + + rd_kafka_dbg( + mshgrp->cluster->rk, MOCK, "MOCK", + "Target assignment set for member %s: %d partition(s)", + member_id, member->assignment->cnt); + } + + /* Bump the epochs */ + TAILQ_FOREACH(member, &mshgrp->members, link) { + member->previous_member_epoch = member->member_epoch; + member->member_epoch = ++mshgrp->group_epoch; + } +} + +/** + * @brief Manual target assignment interface for sharegroups. + */ +void rd_kafka_mock_sharegroup_target_assignment( + rd_kafka_mock_cluster_t *mcluster, + const char *group_id, + const char **member_ids, + rd_kafka_topic_partition_list_t **assignment, + size_t member_cnt) { + rd_kafka_mock_sharegroup_t *mshgrp; + rd_kafka_mock_sharegroup_target_assignment_t *target_assignment; + size_t i; + rd_kafkap_str_t *group_id_str; + + mtx_lock(&mcluster->lock); + group_id_str = rd_kafkap_str_new(group_id, -1); + mshgrp = rd_kafka_mock_sharegroup_find(mcluster, group_id_str); + rd_kafkap_str_destroy(group_id_str); + + if (!mshgrp) { + rd_kafka_log(mcluster->rk, LOG_ERR, "MOCK", + "Sharegroup %s not found for target assignment", + group_id); + mtx_unlock(&mcluster->lock); + return; + } + + mshgrp->manual_assignment = rd_true; + target_assignment = rd_kafka_mock_sharegroup_target_assignment_new(); + + for (i = 0; i < member_cnt; i++) { + rd_list_add(&target_assignment->member_ids, + rd_strdup(member_ids[i])); + rd_list_add(&target_assignment->assignment, + rd_kafka_topic_partition_list_copy(assignment[i])); + } + rd_kafka_mock_sharegroup_target_assignment_set(mshgrp, + target_assignment); + rd_kafka_mock_sharegroup_target_assignment_destroy(target_assignment); + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the sharegroup session timeout for the sharegroup. + */ +void rd_kafka_mock_sharegroup_set_session_timeout( + rd_kafka_mock_cluster_t *mcluster, + int session_timeout_ms) { + mtx_lock(&mcluster->lock); + mcluster->defaults.sharegroup_session_timeout_ms = session_timeout_ms; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the sharegroup heartbeat interval for the sharegroup. + */ +void rd_kafka_mock_sharegroup_set_heartbeat_interval( + rd_kafka_mock_cluster_t *mcluster, + int heartbeat_interval_ms) { + mtx_lock(&mcluster->lock); + mcluster->defaults.sharegroup_heartbeat_interval_ms = + heartbeat_interval_ms; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the maximum delivery attempts per record for the sharegroup. + */ +void rd_kafka_mock_sharegroup_set_max_delivery_attempts( + rd_kafka_mock_cluster_t *mcluster, + int max_attempts) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->max_delivery_attempts = max_attempts; + mcluster->defaults.sharegroup_max_delivery_attempts = max_attempts; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the per-record lock duration in milliseconds for the sharegroup. + */ +void rd_kafka_mock_sharegroup_set_record_lock_duration( + rd_kafka_mock_cluster_t *mcluster, + int lock_duration_ms) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->record_lock_duration_ms = lock_duration_ms; + mcluster->defaults.sharegroup_record_lock_duration_ms = + lock_duration_ms; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the share group isolation level for transactions. + */ +void rd_kafka_mock_sharegroup_set_isolation_level( + rd_kafka_mock_cluster_t *mcluster, + int level) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->isolation_level = level; + mcluster->defaults.sharegroup_isolation_level = level; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the maximum number of members allowed in a share group. + */ +void rd_kafka_mock_sharegroup_set_max_size(rd_kafka_mock_cluster_t *mcluster, + int max_size) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->max_size = max_size; + mcluster->defaults.sharegroup_max_size = max_size; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the maximum number of fetch sessions allowed per broker. + */ +void rd_kafka_mock_sharegroup_set_max_fetch_sessions( + rd_kafka_mock_cluster_t *mcluster, + int max_fetch_sessions) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->max_fetch_sessions = max_fetch_sessions; + mcluster->defaults.sharegroup_max_fetch_sessions = max_fetch_sessions; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the maximum number of in-flight record locks per + * share-partition. + */ +void rd_kafka_mock_sharegroup_set_max_record_locks( + rd_kafka_mock_cluster_t *mcluster, + int max_record_locks) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->max_record_locks = max_record_locks; + mcluster->defaults.sharegroup_max_record_locks = max_record_locks; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Set the auto offset reset policy for share groups. + */ +void rd_kafka_mock_sharegroup_set_auto_offset_reset( + rd_kafka_mock_cluster_t *mcluster, + int auto_offset_reset) { + rd_kafka_mock_sharegroup_t *mshgrp; + mtx_lock(&mcluster->lock); + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) + mshgrp->auto_offset_reset = auto_offset_reset; + mcluster->defaults.sharegroup_auto_offset_reset = auto_offset_reset; + mtx_unlock(&mcluster->lock); +} + +/** + * @brief Destroy share fetch session. + */ +void rd_kafka_mock_sgrp_fetch_session_destroy( + rd_kafka_mock_sgrp_fetch_session_t *session) { + rd_free(session->member_id); + RD_IF_FREE(session->partitions, rd_kafka_topic_partition_list_destroy); + rd_free(session); +} + + +/** + * @brief Common share-session validation for ShareFetch / ShareAcknowledge. + * + * Performs: + * 1. Session lookup by (MemberId, NodeId). + * 2. SessionEpoch == 0 : destroy old session if any (caller creates new). + * 3. SessionEpoch == -1 : keep session alive (FinalContext behaviour). + * 4. SessionEpoch > 0 : validate that session exists and epoch matches. + * + * On return, \p *sessionp is set to the looked-up session (or NULL if + * no session was found or it was closed). + * + * @note Does NOT handle SessionEpoch == 0 (open new session), which is + * ShareFetch-specific. The caller is also responsible for + * incrementing session_epoch and updating ts_last_activity on + * success. + * + * @param sgrp Share group. + * @param MemberId Member identifier from the request. + * @param NodeId Node ID of the broker handling the request. + * @param SessionEpoch Session epoch from the request. + * @param sessionp [out] Session pointer. + * @param api_name API name for debug messages ("ShareFetch" etc.). + * + * @returns Error code, or RD_KAFKA_RESP_ERR_NO_ERROR on success. + * + * @locks mcluster->lock MUST be held. + */ +rd_kafka_resp_err_t rd_kafka_mock_sgrp_session_validate( + rd_kafka_mock_sharegroup_t *sgrp, + const rd_kafkap_str_t *MemberId, + int32_t NodeId, + int32_t SessionEpoch, + rd_kafka_mock_sgrp_fetch_session_t **sessionp, + const char *api_name) { + rd_kafka_mock_sgrp_fetch_session_t *session = NULL; + + *sessionp = NULL; + + /* The real Kafka broker's partition leader does NOT + * validate group membership on ShareFetch — share sessions are + * managed independently of the group coordinator. */ + + /* 1. Look up existing session by (MemberId, NodeId). + * In real Kafka, ShareSessionCache is per-broker, so each + * broker maintains its own independent session for the same + * member. We emulate this by keying on both fields. */ + TAILQ_FOREACH(session, &sgrp->fetch_sessions, link) { + if (!rd_kafkap_str_cmp_str(MemberId, session->member_id) && + session->node_id == NodeId) + break; + } + + /* 2. SessionEpoch == 0: open a new session. + * If an old session exists for this member (e.g. after a + * LEAVE→rejoin cycle), destroy it so the caller creates a + * fresh one. */ + if (SessionEpoch == 0) { + if (session) { + rd_kafka_mock_sgrp_release_session_locks(sgrp, session); + TAILQ_REMOVE(&sgrp->fetch_sessions, session, link); + sgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + session = NULL; + } + } else if (SessionEpoch == -1) { + /* 3. SessionEpoch == -1 (FINAL_EPOCH): return the existing + * session so the caller can process final acks and + * then close it. If no session exists, fail with + * SHARE_SESSION_NOT_FOUND per the protocol. */ + if (!session) { + *sessionp = NULL; + return RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND; + } + } else if (SessionEpoch > 0) { + /* 4. SessionEpoch > 0: validate epoch. */ + if (!session) { + /* Session not in cache → + * SHARE_SESSION_NOT_FOUND (distinct from epoch + * mismatch which uses + * INVALID_SHARE_SESSION_EPOCH). */ + *sessionp = NULL; + return RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND; + } else if (SessionEpoch != session->session_epoch) { + /* Epoch mismatch → destroy the stale + * session and return INVALID_SHARE_SESSION_EPOCH. + * The client handles this by resetting its + * per-broker epoch to 0 (opening a fresh session + * on the next fetch). */ + rd_kafka_mock_sgrp_release_session_locks(sgrp, session); + TAILQ_REMOVE(&sgrp->fetch_sessions, session, link); + sgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + *sessionp = NULL; + return RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH; + } + } + + *sessionp = session; + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + + +/** + * @brief Release a single ACQUIRED record state back to AVAILABLE or ARCHIVED. + * + * If \p mshgrp has a max_delivery_attempts limit and the record's + * delivery_count has reached it, the record is archived instead of + * made available again. Clears owner_member_id and lock_expiry_ts. + * + * @locks mcluster->lock MUST be held. + */ +void rd_kafka_mock_sgrp_record_release( + rd_kafka_mock_sharegroup_t *mshgrp, + rd_kafka_mock_sgrp_partmeta_t *pmeta, + rd_kafka_mock_sgrp_record_state_t *state) { + if (state->state == RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED) + pmeta->acquired_cnt--; + if (mshgrp->max_delivery_attempts > 0 && + state->delivery_count >= mshgrp->max_delivery_attempts) { + state->state = RD_KAFKA_MOCK_SGRP_RECORD_ARCHIVED; + } else { + state->state = RD_KAFKA_MOCK_SGRP_RECORD_AVAILABLE; + } + rd_free(state->owner_member_id); + state->owner_member_id = NULL; + state->lock_expiry_ts = 0; +} + +/** + * @brief Release all ACQUIRED records owned by \p session's member on + * the partitions belonging to \p session. + * + * Share sessions are per-broker: closing one broker's session must + * only release the records acquired through that session (the + * partitions that broker leads), never the member's locks held + * through other brokers' intact sessions. + * + * This is called when a session is closed (epoch=-1), replaced + * (epoch=0), invalidated (epoch mismatch), times out, or its + * connection is closed. + * + * @locks mcluster->lock MUST be held. + */ +void rd_kafka_mock_sgrp_release_session_locks( + rd_kafka_mock_sharegroup_t *mshgrp, + const rd_kafka_mock_sgrp_fetch_session_t *session) { + rd_kafka_mock_sgrp_partmeta_t *pmeta; + + if (!session->partitions) + return; + + TAILQ_FOREACH(pmeta, &mshgrp->partitions, link) { + rd_kafka_mock_sgrp_record_state_t *state, *tmp; + rd_kafka_mock_topic_t *mtopic; + rd_kafka_mock_partition_t *mpart; + + if (rd_kafka_topic_partition_list_find_idx_by_id( + session->partitions, pmeta->topic_id, + pmeta->partition) < 0) + continue; + + /* A session can only release locks for partitions its + * broker currently leads: a session's partition list + * can be stale after a leader change, while the locks + * now belong to the new leader's session. */ + mtopic = rd_kafka_mock_topic_find_by_id(mshgrp->cluster, + pmeta->topic_id); + mpart = mtopic ? rd_kafka_mock_partition_find(mtopic, + pmeta->partition) + : NULL; + if (!mpart || !mpart->leader || + mpart->leader->id != session->node_id) + continue; + + TAILQ_FOREACH_SAFE(state, &pmeta->inflight, link, tmp) { + if (state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED) + continue; + if (!state->owner_member_id) + continue; + if (strcmp(state->owner_member_id, + session->member_id) != 0) + continue; + + rd_kafka_mock_sgrp_record_release(mshgrp, pmeta, state); + } + } +} + +/** + * @brief Proactively release any expired acquisition locks. + * + * Iterates all partition metadata in the share group and flips + * ACQUIRED records whose lock_expiry_ts has passed back to AVAILABLE. + * + * @locks mcluster->lock MUST be held. + */ +static void rd_kafka_mock_sgrp_expire_locks(rd_kafka_mock_sharegroup_t *mshgrp, + rd_ts_t now) { + rd_kafka_mock_sgrp_partmeta_t *pmeta; + + TAILQ_FOREACH(pmeta, &mshgrp->partitions, link) { + rd_kafka_mock_sgrp_record_state_t *state, *tmp; + TAILQ_FOREACH_SAFE(state, &pmeta->inflight, link, tmp) { + if (state->state != RD_KAFKA_MOCK_SGRP_RECORD_ACQUIRED) + continue; + if (!state->lock_expiry_ts || + state->lock_expiry_ts > now) + continue; + + /* Lock has expired. If delivery + * count has reached the limit, archive the + * record instead of making it available. */ + rd_kafka_mock_sgrp_record_release(mshgrp, pmeta, state); + } + } +} + +/** + * @brief Periodic timer: expire stale share-fetch sessions and + * proactively reclaim expired acquisition locks. + * + * @locks mcluster->lock is acquired and released. + */ +void rd_kafka_mock_sgrp_fetch_session_tmr_cb(rd_kafka_timers_t *rkts, + void *arg) { + rd_kafka_mock_sharegroup_t *mshgrp = arg; + rd_kafka_mock_sgrp_fetch_session_t *session, *tmp; + rd_ts_t now = rd_clock(); + rd_kafka_mock_cluster_t *mcluster = mshgrp->cluster; + + (void)rkts; + + mtx_lock(&mcluster->lock); + + /* 1. Expire stale sessions and release their member locks. */ + TAILQ_FOREACH_SAFE(session, &mshgrp->fetch_sessions, link, tmp) { + if (session->ts_last_activity + + (mshgrp->session_timeout_ms * 1000) > + now) + continue; + + /* Release all locks held by this member before + * destroying the session. */ + rd_kafka_mock_sgrp_release_session_locks(mshgrp, session); + + TAILQ_REMOVE(&mshgrp->fetch_sessions, session, link); + mshgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + } + + /* 2. Proactively reclaim any expired acquisition locks. + * This catches records whose owning consumer crashed + * without closing its session cleanly. */ + rd_kafka_mock_sgrp_expire_locks(mshgrp, now); + + mtx_unlock(&mcluster->lock); +} + +/** + * @brief A client connection closed, check if any sharegroup has any + * state for this connection that needs to be cleared. + * + * @param mcluster Cluster to search in. + * @param mconn Connection that was closed. + * + * @locks mcluster->lock MUST be held. + */ +void rd_kafka_mock_sharegrps_connection_closed( + rd_kafka_mock_cluster_t *mcluster, + rd_kafka_mock_connection_t *mconn) { + rd_kafka_mock_sharegroup_t *mshgrp; + + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) { + rd_kafka_mock_sharegroup_member_t *member; + /* Clear heartbeat connection for any member on this conn. */ + TAILQ_FOREACH(member, &mshgrp->members, link) { + if (member->conn == mconn) + member->conn = NULL; + } + } +} + +/** + * @brief Close all share fetch sessions on \p node_id. + * + * Called from rd_kafka_mock_connection_close() where mconn->broker is + * guaranteed to be valid. Must NOT be called with a fake connection pointer. + * + * @locks mcluster->lock MUST be held. + */ +void rd_kafka_mock_sharegrps_node_connection_closed( + rd_kafka_mock_cluster_t *mcluster, + int32_t node_id) { + rd_kafka_mock_sharegroup_t *mshgrp; + + TAILQ_FOREACH(mshgrp, &mcluster->sharegrps, link) { + rd_kafka_mock_sgrp_fetch_session_t *session, *tmp; + /* When a connection is disconnected, any share session + * on that broker is automatically closed. */ + TAILQ_FOREACH_SAFE(session, &mshgrp->fetch_sessions, link, + tmp) { + if (session->node_id != node_id) + continue; + rd_kafka_mock_sgrp_release_session_locks(mshgrp, + session); + TAILQ_REMOVE(&mshgrp->fetch_sessions, session, link); + mshgrp->fetch_session_cnt--; + rd_kafka_mock_sgrp_fetch_session_destroy(session); + } + } +} + +/** + * @brief Retrieve the member IDs from a sharegroup. + * + * @param mcluster Mock cluster instance. + * @param group_id The sharegroup ID. + * @param member_ids_out Output array of member IDs (caller must free each + * string and the array itself). + * @param member_cnt_out Output count of members. + * + * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, + * RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND if sharegroup not found. + */ +rd_kafka_resp_err_t +rd_kafka_mock_sharegroup_get_member_ids(rd_kafka_mock_cluster_t *mcluster, + const char *group_id, + char ***member_ids_out, + size_t *member_cnt_out) { + rd_kafka_mock_sharegroup_t *mshgrp; + rd_kafka_mock_sharegroup_member_t *member; + rd_kafkap_str_t *group_id_str; + char **member_ids; + size_t i; + + mtx_lock(&mcluster->lock); + group_id_str = rd_kafkap_str_new(group_id, -1); + mshgrp = rd_kafka_mock_sharegroup_find(mcluster, group_id_str); + rd_kafkap_str_destroy(group_id_str); + + if (!mshgrp) { + mtx_unlock(&mcluster->lock); + *member_ids_out = NULL; + *member_cnt_out = 0; + return RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND; + } + + *member_cnt_out = mshgrp->member_cnt; + if (mshgrp->member_cnt == 0) { + mtx_unlock(&mcluster->lock); + *member_ids_out = NULL; + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + + member_ids = rd_malloc(sizeof(*member_ids) * mshgrp->member_cnt); + i = 0; + TAILQ_FOREACH(member, &mshgrp->members, link) { + member_ids[i++] = rd_strdup(member->id); + } + + mtx_unlock(&mcluster->lock); + *member_ids_out = member_ids; + return RD_KAFKA_RESP_ERR_NO_ERROR; +} \ No newline at end of file diff --git a/lib/librdkafka-2.10.1/src/rdkafka_msg.c b/lib/librdkafka-2.15.0/src/rdkafka_msg.c similarity index 98% rename from lib/librdkafka-2.10.1/src/rdkafka_msg.c rename to lib/librdkafka-2.15.0/src/rdkafka_msg.c index 7c8204dcc79..34a2f54eb85 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_msg.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_msg.c @@ -491,6 +491,8 @@ rd_kafka_produceva(rd_kafka_t *rk, const rd_kafka_vu_t *vus, size_t cnt) { rd_kafka_err2str(err)); goto err; } + /* 'hdrs' is now owned by 'rkm' */ + hdrs = NULL; /* Partition the message */ err = rd_kafka_msg_partitioner(rkt, rkm, 1); @@ -1301,6 +1303,22 @@ rd_kafka_message_t *rd_kafka_message_new(void) { } +/** + * @brief Allocate an rd_kafka_messages_t whose flex \c elems[] array has + * room for exactly \p cnt message pointers, with \c cnt populated. + * + * The caller is responsible for filling \c elems[0..cnt). Internal helper + * used by the share-consumer queue serve to allocate the caller-visible + * messages handle in a single step (no intermediate buffer). + */ +rd_kafka_messages_t *rd_kafka_messages_new(size_t cnt) { + rd_kafka_messages_t *messages = + rd_calloc(1, (sizeof(*messages) + cnt * sizeof(*messages->elems))); + messages->cnt = cnt; + return messages; +} + + /** * @brief Set up a rkmessage from an rko for passing to the application. * @remark Will trigger on_consume() interceptors if any. @@ -1590,6 +1608,20 @@ int32_t rd_kafka_message_leader_epoch(const rd_kafka_message_t *rkmessage) { return rkm->rkm_u.consumer.leader_epoch; } +int16_t rd_kafka_message_delivery_count(const rd_kafka_message_t *rkmessage) { + rd_kafka_msg_t *rkm; + + if (unlikely(!rkmessage)) + return 0; + + rkm = rd_kafka_message2msg((rd_kafka_message_t *)rkmessage); + + if (unlikely(!rkm)) + return 0; + + return rkm->rkm_u.consumer.delivery_count; +} + void rd_kafka_msgq_dump(FILE *fp, const char *what, rd_kafka_msgq_t *rkmq) { rd_kafka_msg_t *rkm; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_msg.h b/lib/librdkafka-2.15.0/src/rdkafka_msg.h similarity index 95% rename from lib/librdkafka-2.10.1/src/rdkafka_msg.h rename to lib/librdkafka-2.15.0/src/rdkafka_msg.h index 063fe96b95c..6bd939d03e0 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_msg.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_msg.h @@ -31,6 +31,7 @@ #include "rdkafka_proto.h" #include "rdkafka_header.h" +#include "rdkafka_share_acknowledgement.h" /** @@ -154,6 +155,20 @@ typedef struct rd_kafka_msg_s { * protocol msg */ int32_t leader_epoch; /**< Leader epoch at the time * the message was fetched. */ + int32_t wire_size; /**< wire-format size of this + * record including the + * length-prefix varint and all + * per-record framing. + * Used by share consumer + * fetch.size telemetry. + */ + rd_kafka_share_internal_acknowledgement_type + ack_type; /**< Share consumer: acknowledgement + * type + * (rd_kafka_share_internal_acknowledgement_type). + * Set during response processing. */ + int16_t delivery_count; /**< Share consumer: delivery + * count. */ } consumer; } rkm_u; } rd_kafka_msg_t; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_msgbatch.h b/lib/librdkafka-2.15.0/src/rdkafka_msgbatch.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_msgbatch.h rename to lib/librdkafka-2.15.0/src/rdkafka_msgbatch.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_msgset.h b/lib/librdkafka-2.15.0/src/rdkafka_msgset.h similarity index 92% rename from lib/librdkafka-2.10.1/src/rdkafka_msgset.h rename to lib/librdkafka-2.15.0/src/rdkafka_msgset.h index ee897b35bd5..22f73857cb2 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_msgset.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_msgset.h @@ -78,6 +78,13 @@ rd_kafka_msgset_parse(rd_kafka_buf_t *rkbuf, rd_kafka_aborted_txns_t *aborted_txns, const struct rd_kafka_toppar_ver *tver); +rd_kafka_resp_err_t +rd_kafka_share_msgset_parse(rd_kafka_buf_t *rkbuf, + rd_kafka_toppar_t *rktp, + rd_kafka_aborted_txns_t *aborted_txns, + const struct rd_kafka_toppar_ver *tver, + rd_kafka_q_t *response_message_q); + #if WITH_ZLIB rd_kafka_resp_err_t rd_kafka_gzip_compress(rd_kafka_broker_t *rkb, int comp_level, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_msgset_reader.c b/lib/librdkafka-2.15.0/src/rdkafka_msgset_reader.c similarity index 77% rename from lib/librdkafka-2.10.1/src/rdkafka_msgset_reader.c rename to lib/librdkafka-2.15.0/src/rdkafka_msgset_reader.c index 451dd35442b..787b3c20ab1 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_msgset_reader.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_msgset_reader.c @@ -517,14 +517,65 @@ rd_kafka_msgset_reader_decompress(rd_kafka_msgset_reader_t *msetr, err: /* Enqueue error messsage: * Create op and push on temporary queue. */ - rd_kafka_consumer_err( - &msetr->msetr_rkq, msetr->msetr_broker_id, err, - msetr->msetr_tver->version, NULL, rktp, Offset, - "Decompression (codec 0x%x) of message at %" PRIu64 " of %" PRIusz - " bytes failed: %s", - codec, Offset, compressed_size, rd_kafka_err2str(err)); + if (!RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) { + /* Regular consumer */ + rd_kafka_consumer_err( + &msetr->msetr_rkq, msetr->msetr_broker_id, err, + msetr->msetr_tver->version, NULL, rktp, Offset, + "Decompression (codec 0x%x) of message at %" PRIu64 + " of %" PRIusz " bytes failed: %s", + codec, Offset, compressed_size, rd_kafka_err2str(err)); + return err; + } else { + /** + * v0/v1 MessageSet: single error + * TODO KIP-932: Check the handling in case of v0/v1 messages + * for Share Consumers. + */ + if (msetr->msetr_v2_hdr == NULL) { + rd_kafka_consumer_err( + &msetr->msetr_rkq, msetr->msetr_broker_id, err, + msetr->msetr_tver->version, NULL, rktp, Offset, + "Decompression (codec 0x%x) of message at %" PRIu64 + " of %" PRIusz " bytes failed: %s", + codec, Offset, compressed_size, + rd_kafka_err2str(err)); + } else { + /* Share consumer with MessageSet v2: report error for + * the entire batch with RELEASE acknowledgement — + * decompression capability varies by consumer, so + * another consumer might succeed on the same batch. */ + int64_t LastOffset = + msetr->msetr_v2_hdr->BaseOffset + + msetr->msetr_v2_hdr->LastOffsetDelta; + + /** + * TODO KIP-932: Add a TODO to translate errors inside + * this function. _BAD_COMPRESSION and _BAD_MSG should + * be sent as INVALID_MSG, and _CRIT_SYS_RESOURCE should + * be sent as _CRIT_SYS_RESOURCE. + */ + rd_kafka_share_msgset_err_ops( + &msetr->msetr_rkq, msetr->msetr_broker_id, err, + msetr->msetr_tver->version, rktp, + msetr->msetr_v2_hdr->BaseOffset, LastOffset, + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "Decompression (codec 0x%x) of MessageSet at " + "offsets " + "%" PRId64 "-%" PRId64 " of %" PRIusz + " bytes failed: %s", + codec, msetr->msetr_v2_hdr->BaseOffset, LastOffset, + compressed_size, rd_kafka_err2str(err)); + } + } - return err; + /** + * In case of Share Consumers the errors are propagated + * as ops on the queue, but we return no error to make sure the + * MessageSet parser continues parsing the next messagesets, which might + * succeed even if this one failed. + */ + return RD_KAFKA_RESP_ERR_NO_ERROR; } @@ -535,6 +586,8 @@ rd_kafka_msgset_reader_decompress(rd_kafka_msgset_reader_t *msetr, * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or on single-message errors, * or any other error code when the MessageSet parser should stop * parsing (such as for partial Messages). + * TODO KIP-932: We might need to check the handling of error ops + * for share consumers with v0/v1 messagesets. */ static rd_kafka_resp_err_t rd_kafka_msgset_reader_msg_v0_1(rd_kafka_msgset_reader_t *msetr) { @@ -653,7 +706,8 @@ rd_kafka_msgset_reader_msg_v0_1(rd_kafka_msgset_reader_t *msetr) { * we cant perform this offset check here * in that case. */ if (!relative_offsets && - hdr.Offset < rktp->rktp_offsets.fetch_pos.offset) + hdr.Offset < rktp->rktp_offsets.fetch_pos.offset && + !RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */ /* Handle compressed MessageSet */ @@ -734,8 +788,14 @@ rd_kafka_msgset_reader_msg_v2(rd_kafka_msgset_reader_t *msetr) { ? LOG_DEBUG : 0; size_t message_end; + size_t record_start_pos; rd_kafka_fetch_pos_t msetr_pos; + /* Capture buffer position before reading length varint so we can + * compute the total wire-format size of this record (length varint + * size + body bytes) for share consumer fetch.size telemetry. */ + record_start_pos = rd_slice_offset(&rkbuf->rkbuf_reader); + rd_kafka_buf_read_varint(rkbuf, &hdr.Length); message_end = rd_slice_offset(&rkbuf->rkbuf_reader) + (size_t)hdr.Length; @@ -751,8 +811,11 @@ rd_kafka_msgset_reader_msg_v2(rd_kafka_msgset_reader_t *msetr) { * epoch is different the fetch will fail (KIP-320) and if offset leader * epoch is different it'll return an empty fetch (KIP-595). If we * checked it, it's possible to have a loop when moving from a broker - * that supports leader epoch to one that doesn't. */ - if (hdr.Offset < rktp->rktp_offsets.fetch_pos.offset) { + * that supports leader epoch to one that doesn't. + * Share consumers may receive re-delivered records at earlier offsets + * after RELEASE, so skip this check for them. */ + if (hdr.Offset < rktp->rktp_offsets.fetch_pos.offset && + !RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) { rd_rkb_dbg( msetr->msetr_rkb, MSG, "MSG", "%s [%" PRId32 @@ -765,7 +828,11 @@ rd_kafka_msgset_reader_msg_v2(rd_kafka_msgset_reader_t *msetr) { return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */ } - /* Handle control messages */ + /** + * Handle control messages + * TODO KIP-932: GA: This branch shouldn't hit for share consumers, + * as we skip control messages for share consumers. + */ if (msetr->msetr_v2_hdr->Attributes & RD_KAFKA_MSGSET_V2_ATTR_CONTROL) { struct { int64_t KeySize; @@ -951,6 +1018,12 @@ rd_kafka_msgset_reader_msg_v2(rd_kafka_msgset_reader_t *msetr) { } + /* Cache the wire-format size of this record (length varint + + * body) on the rkm. + * Used by share consumer fetch.size telemetry. */ + rkm->rkm_u.consumer.wire_size = + (int32_t)(message_end - record_start_pos); + /* Enqueue message on temporary queue */ rd_kafka_q_enq(&msetr->msetr_rkq, rko); msetr->msetr_msgcnt++; @@ -1084,19 +1157,89 @@ rd_kafka_msgset_reader_v2(rd_kafka_msgset_reader_t *msetr) { if (unlikely((uint32_t)hdr.Crc != calc_crc)) { /* Propagate CRC error to application and * continue with next message. */ - rd_kafka_consumer_err( - &msetr->msetr_rkq, msetr->msetr_broker_id, - RD_KAFKA_RESP_ERR__BAD_MSG, - msetr->msetr_tver->version, NULL, rktp, - hdr.BaseOffset, - "MessageSet at offset %" PRId64 " (%" PRId32 - " bytes) " - "failed CRC32C check " - "(original 0x%" PRIx32 - " != " - "calculated 0x%" PRIx32 ")", - hdr.BaseOffset, hdr.Length, hdr.Crc, calc_crc); - rd_kafka_buf_skip_to(rkbuf, crc_len); + if (!RD_KAFKA_IS_SHARE_CONSUMER( + msetr->msetr_rkb->rkb_rk)) { + rd_kafka_consumer_err( + &msetr->msetr_rkq, msetr->msetr_broker_id, + RD_KAFKA_RESP_ERR__BAD_MSG, + msetr->msetr_tver->version, NULL, rktp, + hdr.BaseOffset, + "MessageSet at offset %" PRId64 " (%" PRId32 + " bytes) " + "failed CRC32C check " + "(original 0x%" PRIx32 + " != " + "calculated 0x%" PRIx32 ")", + hdr.BaseOffset, hdr.Length, hdr.Crc, + calc_crc); + /* Skip to end of MessageSet */ + /** + * FIXME: `crc_len` is a byte *length* + * (Length - 4 - 1 - 4), but + * rd_kafka_buf_skip_to() interprets its + * argument as an absolute slice position. + * The current call leaves the reader 21 + * bytes short of the end of the MessageSet + * (when the slice base offset is 0), and + * for any subsequent MessageSet within + * the same slice the size_t subtraction + * inside skip_to underflows and triggers + * a spurious __UNDERFLOW. The intended + * target is end-of-MessageSet, i.e. + * `len_start + hdr.Length` — see the + * share-consumer branch below which does + * this correctly. Replace with: + * rd_kafka_buf_skip_to(rkbuf, + * len_start + hdr.Length); + */ + rd_kafka_buf_skip_to(rkbuf, crc_len); + + } else { + int16_t Attributes; + int32_t LastOffsetDelta; + + /* Read LastOffsetDelta to + * determine full offset range. + * + * TODO KIP-932: These bytes come from + * within the CRC-covered region of the + * MessageSet body, and the CRC has + * just failed — so `Attributes` and + * `LastOffsetDelta` may be corrupt. A + * garbage `LastOffsetDelta` can: + * - be huge → fan-out emits millions of + * error ops for offsets that don't + * exist (memory/queue blowup); + * - be negative → `LastOffset < + * BaseOffset`, fan-out loop is a + * silent no-op (no error surfaced); + * - look valid but be off → REJECT ops + * issued for the wrong offset range. + */ + rd_kafka_buf_read_i16(rkbuf, &Attributes); + rd_kafka_buf_read_i32(rkbuf, &LastOffsetDelta); + LastOffset = hdr.BaseOffset + LastOffsetDelta; + + rd_kafka_share_msgset_err_ops( + &msetr->msetr_rkq, msetr->msetr_broker_id, + RD_KAFKA_RESP_ERR__BAD_MSG, + msetr->msetr_tver->version, rktp, + hdr.BaseOffset, LastOffset, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "MessageSet at offsets %" PRId64 "-%" PRId64 + " (%" PRId32 + " bytes) " + "failed CRC32C check " + "(original 0x%" PRIx32 + " != " + "calculated 0x%" PRIx32 ")", + hdr.BaseOffset, LastOffset, hdr.Length, + hdr.Crc, calc_crc); + /* Skip to end of MessageSet */ + rd_kafka_buf_skip_to(rkbuf, + len_start + hdr.Length); + } + rd_atomic64_add(&msetr->msetr_rkb->rkb_c.rx_err, 1); return RD_KAFKA_RESP_ERR_NO_ERROR; } @@ -1125,8 +1268,11 @@ rd_kafka_msgset_reader_v2(rd_kafka_msgset_reader_t *msetr) { rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, hdr.BaseOffset, payload_size); - /* If entire MessageSet contains old outdated offsets, skip it. */ - if (LastOffset < rktp->rktp_offsets.fetch_pos.offset) { + /* If entire MessageSet contains old outdated offsets, skip it. + * Share consumers may receive re-delivered records at earlier offsets + * after RELEASE, so skip this check for them. */ + if (LastOffset < rktp->rktp_offsets.fetch_pos.offset && + !RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) { rd_kafka_buf_skip(rkbuf, payload_size); goto done; } @@ -1177,7 +1323,8 @@ rd_kafka_msgset_reader_v2(rd_kafka_msgset_reader_t *msetr) { * to avoid getting stuck on compacted MessageSets where the last * Message in the MessageSet has an Offset < MessageSet header's * last offset. See KAFKA-5443 */ - msetr->msetr_next_offset = LastOffset + 1; + if (!RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) + msetr->msetr_next_offset = LastOffset + 1; msetr->msetr_v2_hdr = NULL; @@ -1236,26 +1383,73 @@ rd_kafka_msgset_reader_peek_msg_version(rd_kafka_msgset_reader_t *msetr, (int)*MagicBytep, Offset, read_offset, rd_slice_size(&rkbuf->rkbuf_reader)); - if (Offset >= - msetr->msetr_rktp->rktp_offsets.fetch_pos.offset) { - rd_kafka_consumer_err( + if (!RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) { + if (Offset >= + msetr->msetr_rktp->rktp_offsets.fetch_pos.offset) { + rd_kafka_consumer_err( + &msetr->msetr_rkq, msetr->msetr_broker_id, + RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, + msetr->msetr_tver->version, NULL, rktp, + Offset, + "Unsupported Message(Set) MagicByte %d " + "at offset %" PRId64, + (int)*MagicBytep, Offset); + /* Skip message(set) */ + msetr->msetr_rktp->rktp_offsets.fetch_pos + .offset = Offset + 1; + } + + /* Skip this Message(Set). + * If the message is malformed, the skip may trigger + * err_parse and return ERR__BAD_MSG. */ + rd_kafka_buf_read_i32(rkbuf, &Length); + rd_kafka_buf_skip(rkbuf, Length); + } else { + /* Share consumer: peek further into the MessageSet v2 + * header so we can compute the full offset range, + * emit a REJECT range-error op, and skip the + * remainder. Speculatively assumes the v2 header + * layout is preserved for any future MessageSet + * version above v2 (the only versions reaching this + * branch — v0/v1 are filtered by the MagicByte + * bounds check above). + * TODO KIP-932: Check if this handling is correct as + * we dont know the magic byte, and we are assuming + * its v2.*/ + int32_t PartitionLeaderEpoch; + int8_t MagicByteActual; + int32_t Crc; + int16_t Attributes; + int32_t LastOffsetDelta; + int64_t LastOffset; + /* Bytes consumed below from the MessageSet body + * (everything counted by Length): PartitionLeaderEpoch + * (4) + MagicByte (1) + Crc (4) + Attributes (2) + + * LastOffsetDelta (4). */ + const size_t header_bytes_read = 4 + 1 + 4 + 2 + 4; + + rd_kafka_buf_read_i32(rkbuf, &Length); + rd_kafka_buf_read_i32(rkbuf, &PartitionLeaderEpoch); + rd_kafka_buf_read_i8(rkbuf, &MagicByteActual); + rd_kafka_buf_read_i32(rkbuf, &Crc); + rd_kafka_buf_read_i16(rkbuf, &Attributes); + rd_kafka_buf_read_i32(rkbuf, &LastOffsetDelta); + + /* Skip remaining bytes in MessageSet. */ + rd_kafka_buf_skip(rkbuf, Length - header_bytes_read); + + LastOffset = Offset + LastOffsetDelta; + + rd_kafka_share_msgset_err_ops( &msetr->msetr_rkq, msetr->msetr_broker_id, RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, - msetr->msetr_tver->version, NULL, rktp, Offset, - "Unsupported Message(Set) MagicByte %d " - "at offset %" PRId64, - (int)*MagicBytep, Offset); - /* Skip message(set) */ - msetr->msetr_rktp->rktp_offsets.fetch_pos.offset = - Offset + 1; + msetr->msetr_tver->version, rktp, Offset, + LastOffset, RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Unsupported Message(Set) MagicByte %d at offsets " + "%" PRId64 "-%" PRId64, + (int)MagicByteActual, Offset, LastOffset); } - /* Skip this Message(Set). - * If the message is malformed, the skip may trigger err_parse - * and return ERR__BAD_MSG. */ - rd_kafka_buf_read_i32(rkbuf, &Length); - rd_kafka_buf_skip(rkbuf, Length); - return RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED; } @@ -1297,8 +1491,16 @@ rd_kafka_msgset_reader(rd_kafka_msgset_reader_t *msetr) { * due to its use of sendfile(2). */ return RD_KAFKA_RESP_ERR_NO_ERROR; - /* Continue on unsupported MsgVersions, the - * MessageSet will be skipped. */ + /* For share consumer: continue on unsupported + * MsgVersions, the MessageSet will be skipped. Reset + * error so loop continues parsing remaining + * MessageSets. For regular consumer: stop parsing (old + * behavior). */ + if (RD_KAFKA_IS_SHARE_CONSUMER( + msetr->msetr_rkb->rkb_rk) && + err == RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED) { + err = RD_KAFKA_RESP_ERR_NO_ERROR; + } continue; } @@ -1327,6 +1529,14 @@ static void rd_kafka_msgset_reader_postproc(rd_kafka_msgset_reader_t *msetr, if (rko) { *last_offsetp = rko->rko_u.fetch.rkm.rkm_offset; + /* + * TODO KIP-932: Though we shouldn't reach here for share + * consumers as msetr_relative_offsets is set to true only + * for MsgVersion v1 but still check if MsgVersion v1 is + * possible with share consumer. If not, this handling can + * be skipped safely. If yes, then we should use negative + * offset in place of min_offset instead of fetch_pos.offset. + */ if (*last_offsetp != -1 && msetr->msetr_relative_offsets) { /* Update messages to absolute offsets * and purge any messages older than the current @@ -1381,6 +1591,11 @@ rd_kafka_msgset_reader_run(rd_kafka_msgset_reader_t *msetr) { err = RD_KAFKA_RESP_ERR_NO_ERROR; } else if (rktp->rktp_fetch_msg_max_bytes < (1 << 30)) { + /* TODO KIP-932: Skip this branch for share consumers. + * ShareFetch has no per-partition MaxBytes field, so + * growing rktp_fetch_msg_max_bytes never changes what + * the broker returns. + */ rktp->rktp_fetch_msg_max_bytes *= 2; rd_rkb_dbg(msetr->msetr_rkb, FETCH, "CONSUME", "Topic %s [%" PRId32 @@ -1394,15 +1609,35 @@ rd_kafka_msgset_reader_run(rd_kafka_msgset_reader_t *msetr) { err = RD_KAFKA_RESP_ERR_NO_ERROR; } else if (!err && msetr->msetr_aborted_cnt == 0) { - rd_kafka_consumer_err( - &msetr->msetr_rkq, msetr->msetr_broker_id, - RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE, - msetr->msetr_tver->version, NULL, rktp, - rktp->rktp_offsets.fetch_pos.offset, - "Message at offset %" PRId64 - " might be too large to fetch, try increasing " - "receive.message.max.bytes", - rktp->rktp_offsets.fetch_pos.offset); + if (!RD_KAFKA_IS_SHARE_CONSUMER( + msetr->msetr_rkb->rkb_rk)) { + rd_kafka_consumer_err( + &msetr->msetr_rkq, msetr->msetr_broker_id, + RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE, + msetr->msetr_tver->version, NULL, rktp, + rktp->rktp_offsets.fetch_pos.offset, + "Message at offset %" PRId64 + " might be too large to fetch, try " + "increasing receive.message.max.bytes", + rktp->rktp_offsets.fetch_pos.offset); + } else { + /* Share consumer: fetch_pos.offset is not + * meaningful, and the temp_fetchq + * acquired-range filter would drop any op + * without a valid offset. Enqueue the error + * directly onto the cgrp queue served by + * rd_kafka_share_poll so the app + * sees the partition-level failure. */ + rd_kafka_consumer_err( + msetr->msetr_rkb->rkb_rk->rk_cgrp->rkcg_q, + msetr->msetr_broker_id, + RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE, + msetr->msetr_tver->version, NULL, rktp, + RD_KAFKA_OFFSET_INVALID, + "Message might be too large to fetch, " + "try increasing " + "receive.message.max.bytes"); + } } else if (msetr->msetr_aborted_cnt > 0) { /* Noop */ @@ -1422,36 +1657,60 @@ rd_kafka_msgset_reader_run(rd_kafka_msgset_reader_t *msetr) { err = RD_KAFKA_RESP_ERR_NO_ERROR; } - rd_rkb_dbg(msetr->msetr_rkb, MSG | RD_KAFKA_DBG_FETCH, "CONSUME", - "Enqueue %i %smessage(s) (%" PRId64 - " bytes, %d ops) on %s [%" PRId32 - "] fetch queue (qlen %d, v%d, last_offset %" PRId64 - ", %d ctrl msgs, %d aborted msgsets, %s)", - msetr->msetr_msgcnt, msetr->msetr_srcname, - msetr->msetr_msg_bytes, rd_kafka_q_len(&msetr->msetr_rkq), - rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, - rd_kafka_q_len(msetr->msetr_par_rkq), - msetr->msetr_tver->version, last_offset, - msetr->msetr_ctrl_cnt, msetr->msetr_aborted_cnt, - msetr->msetr_compression - ? rd_kafka_compression2str(msetr->msetr_compression) - : "uncompressed"); + if (!RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) { + rd_rkb_dbg( + msetr->msetr_rkb, MSG | RD_KAFKA_DBG_FETCH, "CONSUME", + "Enqueue %i %smessage(s) (%" PRId64 + " bytes, %d ops) on %s [%" PRId32 + "] fetch queue (qlen %d, v%d, last_offset %" PRId64 + ", %d ctrl msgs, %d aborted msgsets, %s)", + msetr->msetr_msgcnt, msetr->msetr_srcname, + msetr->msetr_msg_bytes, rd_kafka_q_len(&msetr->msetr_rkq), + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + rd_kafka_q_len(msetr->msetr_par_rkq), + msetr->msetr_tver->version, last_offset, + msetr->msetr_ctrl_cnt, msetr->msetr_aborted_cnt, + msetr->msetr_compression + ? rd_kafka_compression2str(msetr->msetr_compression) + : "uncompressed"); + } else { + rd_rkb_dbg( + msetr->msetr_rkb, MSG | RD_KAFKA_DBG_FETCH, "CONSUME", + "Enqueue %i %smessage(s) (%" PRId64 + " bytes, %d ops) on %s [%" PRId32 + "] temp fetchq (qlen %d, last_offset %" PRId64 + ", %d ctrl msgs, %s)", + msetr->msetr_msgcnt, msetr->msetr_srcname, + msetr->msetr_msg_bytes, rd_kafka_q_len(&msetr->msetr_rkq), + rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition, + rd_kafka_q_len(msetr->msetr_par_rkq), last_offset, + msetr->msetr_ctrl_cnt, + msetr->msetr_compression + ? rd_kafka_compression2str(msetr->msetr_compression) + : "uncompressed"); + } /* Concat all messages&errors onto the parent's queue * (the partition's fetch queue) */ if (rd_kafka_q_concat(msetr->msetr_par_rkq, &msetr->msetr_rkq) != -1) { /* Update partition's fetch offset based on * last message's offest. */ - if (likely(last_offset != -1)) + if (likely(last_offset != -1) && + !RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) rktp->rktp_offsets.fetch_pos.offset = last_offset + 1; } /* Adjust next fetch offset if outlier code has indicated * an even later next offset. */ - if (msetr->msetr_next_offset > rktp->rktp_offsets.fetch_pos.offset) - rktp->rktp_offsets.fetch_pos.offset = msetr->msetr_next_offset; - - rktp->rktp_offsets.fetch_pos.leader_epoch = msetr->msetr_leader_epoch; + if (!RD_KAFKA_IS_SHARE_CONSUMER(msetr->msetr_rkb->rkb_rk)) { + if (msetr->msetr_next_offset > + rktp->rktp_offsets.fetch_pos.offset) + rktp->rktp_offsets.fetch_pos.offset = + msetr->msetr_next_offset; + + rktp->rktp_offsets.fetch_pos.leader_epoch = + msetr->msetr_leader_epoch; + } rd_kafka_q_destroy_owner(&msetr->msetr_rkq); @@ -1498,6 +1757,40 @@ rd_kafka_msgset_parse(rd_kafka_buf_t *rkbuf, } +/** + * @brief Parse one MessageSet at the current buffer read position, + * enqueueing messages, propagating errors, etc. + * @remark The current rkbuf_reader slice must be limited to the MessageSet size + * + * @returns see rd_kafka_msgset_reader_run() + */ +rd_kafka_resp_err_t +rd_kafka_share_msgset_parse(rd_kafka_buf_t *rkbuf, + rd_kafka_toppar_t *rktp, + rd_kafka_aborted_txns_t *aborted_txns, + const struct rd_kafka_toppar_ver *tver, + rd_kafka_q_t *response_message_q) { + rd_kafka_msgset_reader_t msetr; + rd_kafka_resp_err_t err; + + rd_kafka_msgset_reader_init(&msetr, rkbuf, rktp, tver, aborted_txns, + response_message_q); + + /* Parse and handle the message set */ + err = rd_kafka_msgset_reader_run(&msetr); + + rd_atomic64_add(&rktp->rktp_c.rx_msgs, msetr.msetr_msgcnt); + rd_atomic64_add(&rktp->rktp_c.rx_msg_bytes, msetr.msetr_msg_bytes); + + rd_avg_add(&rktp->rktp_rkt->rkt_avg_batchcnt, + (int64_t)msetr.msetr_msgcnt); + rd_avg_add(&rktp->rktp_rkt->rkt_avg_batchsize, + (int64_t)msetr.msetr_msg_bytes); + + return err; +} + + /** * @brief Offset comparator */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_msgset_writer.c b/lib/librdkafka-2.15.0/src/rdkafka_msgset_writer.c similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_msgset_writer.c rename to lib/librdkafka-2.15.0/src/rdkafka_msgset_writer.c index 3e5ea3b08da..9ff595bfbe0 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_msgset_writer.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_msgset_writer.c @@ -145,12 +145,16 @@ rd_kafka_msgset_writer_select_MsgVersion(rd_kafka_msgset_writer_t *msetw) { * by both client and broker, else disable compression. */ if (msetw->msetw_compression && - (rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_Produce, 0, - compr_req[msetw->msetw_compression].ApiVersion, NULL) == -1 || - (compr_req[msetw->msetw_compression].feature && - !(msetw->msetw_rkb->rkb_features & - compr_req[msetw->msetw_compression].feature)))) { + ( + /* ApiVersion is smaller than the minimum one */ + !rd_kafka_broker_ApiVersion_at_least( + rkb, RD_KAFKAP_Produce, + compr_req[msetw->msetw_compression].ApiVersion) || + /* There's a corresponding feature and it isn't supported + * by this broker. */ + (compr_req[msetw->msetw_compression].feature && + !(msetw->msetw_rkb->rkb_features & + compr_req[msetw->msetw_compression].feature)))) { if (unlikely( rd_interval(&rkb->rkb_suppress.unsupported_compression, /* at most once per day */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_offset.c b/lib/librdkafka-2.15.0/src/rdkafka_offset.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_offset.c rename to lib/librdkafka-2.15.0/src/rdkafka_offset.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_offset.h b/lib/librdkafka-2.15.0/src/rdkafka_offset.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_offset.h rename to lib/librdkafka-2.15.0/src/rdkafka_offset.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_op.c b/lib/librdkafka-2.15.0/src/rdkafka_op.c similarity index 81% rename from lib/librdkafka-2.10.1/src/rdkafka_op.c rename to lib/librdkafka-2.15.0/src/rdkafka_op.c index 5dbbf9c9d4d..9fbad12469d 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_op.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_op.c @@ -36,6 +36,7 @@ #include "rdkafka_proto.h" #include "rdkafka_offset.h" #include "rdkafka_error.h" +#include "rdkafka_share_acknowledgement.h" /* Current number of rd_kafka_op_t */ rd_atomic32_t rd_kafka_op_cnt; @@ -122,7 +123,25 @@ const char *rd_kafka_op2str(rd_kafka_op_type_t type) { "REPLY:RD_KAFKA_OP_SET_TELEMETRY_BROKER", [RD_KAFKA_OP_TERMINATE_TELEMETRY] = "REPLY:RD_KAFKA_OP_TERMINATE_TELEMETRY", - [RD_KAFKA_OP_ELECTLEADERS] = "REPLY:ELECTLEADERS", + [RD_KAFKA_OP_ELECTLEADERS] = "REPLY:ELECTLEADERS", + [RD_KAFKA_OP_SHARE_FETCH] = "REPLY:SHARE_FETCH", + [RD_KAFKA_OP_SHARE_FETCH_FANOUT] = "REPLY:SHARE_FETCH_FANOUT", + [RD_KAFKA_OP_SHARE_SESSION_PARTITION_ADD] = + "REPLY:SHARE_SESSION_PARTITION_ADD", + [RD_KAFKA_OP_SHARE_SESSION_PARTITION_REMOVE] = + "REPLY:SHARE_SESSION_PARTITION_REMOVE", + [RD_KAFKA_OP_SHARE_FETCH_RESPONSE] = "REPLY:SHARE_FETCH_RESPONSE", + [RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE] = + "REPLY:SHARE_ACK_COMMIT_CB_EXECUTE", + [RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER] = + "REPLY:SHARE_ACK_COMMIT_CB_REGISTER", + [RD_KAFKA_OP_SHARE_COMMIT_ASYNC_FANOUT] = + "REPLY:SHARE_COMMIT_ASYNC_FANOUT", + [RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT] = + "REPLY:SHARE_COMMIT_SYNC_FANOUT", + [RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT_REPLY] = + "REPLY:SHARE_COMMIT_SYNC_FANOUT_REPLY", + [RD_KAFKA_OP_SHARE_SESSION_CLEAR] = "REPLY:SHARE_SESSION_CLEAR", }; if (type & RD_KAFKA_OP_REPLY) @@ -287,6 +306,24 @@ rd_kafka_op_t *rd_kafka_op_new0(const char *source, rd_kafka_op_type_t type) { sizeof(rko->rko_u.telemetry_broker), [RD_KAFKA_OP_TERMINATE_TELEMETRY] = _RD_KAFKA_OP_EMPTY, [RD_KAFKA_OP_ELECTLEADERS] = sizeof(rko->rko_u.admin_request), + [RD_KAFKA_OP_SHARE_FETCH] = sizeof(rko->rko_u.share_fetch), + [RD_KAFKA_OP_SHARE_FETCH_FANOUT] = + sizeof(rko->rko_u.share_fetch_fanout), + [RD_KAFKA_OP_SHARE_SESSION_PARTITION_ADD] = _RD_KAFKA_OP_EMPTY, + [RD_KAFKA_OP_SHARE_SESSION_PARTITION_REMOVE] = _RD_KAFKA_OP_EMPTY, + [RD_KAFKA_OP_SHARE_FETCH_RESPONSE] = + sizeof(rko->rko_u.share_fetch_response), + [RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE] = + sizeof(rko->rko_u.share_ack_commit_cb_execute), + [RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER] = + sizeof(rko->rko_u.share_ack_commit_cb_register), + [RD_KAFKA_OP_SHARE_COMMIT_ASYNC_FANOUT] = + sizeof(rko->rko_u.share_commit_async_fanout), + [RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT] = + sizeof(rko->rko_u.share_commit_sync_fanout), + [RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT_REPLY] = + sizeof(rko->rko_u.share_commit_sync_fanout_reply), + [RD_KAFKA_OP_SHARE_SESSION_CLEAR] = _RD_KAFKA_OP_EMPTY, }; size_t tsize = op2size[type & ~RD_KAFKA_OP_FLAGMASK]; @@ -468,6 +505,7 @@ void rd_kafka_op_destroy(rd_kafka_op_t *rko) { case RD_KAFKA_OP_MOCK: RD_IF_FREE(rko->rko_u.mock.name, rd_free); RD_IF_FREE(rko->rko_u.mock.str, rd_free); + RD_IF_FREE(rko->rko_u.mock.errs, rd_free); if (rko->rko_u.mock.metrics) { int64_t i; for (i = 0; i < rko->rko_u.mock.hi; i++) @@ -507,6 +545,62 @@ void rd_kafka_op_destroy(rd_kafka_op_t *rko) { rd_kafka_broker_destroy); break; + case RD_KAFKA_OP_SHARE_FETCH: { + RD_IF_FREE(rko->rko_u.share_fetch.target_broker, + rd_kafka_broker_destroy); + RD_IF_FREE(rko->rko_u.share_fetch.ack_details, rd_list_destroy); + break; + } + + case RD_KAFKA_OP_TERMINATE: + RD_IF_FREE(rko->rko_u.terminated.ack_batches, rd_list_destroy); + break; + + case RD_KAFKA_OP_SHARE_FETCH_FANOUT: + RD_IF_FREE(rko->rko_u.share_fetch_fanout.ack_batches, + rd_list_destroy); + break; + + case RD_KAFKA_OP_SHARE_COMMIT_ASYNC_FANOUT: + RD_IF_FREE(rko->rko_u.share_commit_async_fanout.ack_batches, + rd_list_destroy); + break; + + case RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT: + RD_IF_FREE(rko->rko_u.share_commit_sync_fanout.ack_batches, + rd_list_destroy); + RD_IF_FREE(rko->rko_u.share_commit_sync_fanout.results, + rd_kafka_topic_partition_list_destroy); + break; + + case RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT_REPLY: + RD_IF_FREE(rko->rko_u.share_commit_sync_fanout_reply.results, + rd_kafka_topic_partition_list_destroy); + break; + + case RD_KAFKA_OP_SHARE_FETCH_RESPONSE: { + if (rko->rko_u.share_fetch_response.message_rkos) { + /* Messages not handed to app, destroy them. */ + rd_kafka_op_t *msg_rko; + int mi; + RD_LIST_FOREACH( + msg_rko, + rko->rko_u.share_fetch_response.message_rkos, mi) { + rd_kafka_op_destroy(msg_rko); + } + rd_list_destroy( + rko->rko_u.share_fetch_response.message_rkos); + } + RD_IF_FREE(rko->rko_u.share_fetch_response.inflight_acks, + rd_list_destroy); + break; + } + + case RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE: + RD_IF_FREE(rko->rko_u.share_ack_commit_cb_execute.partitions, + rd_kafka_share_partition_offsets_list_destroy); + break; + default: break; } @@ -564,6 +658,10 @@ void rd_kafka_q_op_err(rd_kafka_q_t *rkq, * * @sa rd_kafka_q_op_err() */ +/** + * TODO KIP-932: GA: Improve handling of this particular function + * as we dont need most information available here. + */ void rd_kafka_consumer_err(rd_kafka_q_t *rkq, int32_t broker_id, rd_kafka_resp_err_t err, @@ -600,6 +698,67 @@ void rd_kafka_consumer_err(rd_kafka_q_t *rkq, } +/** + * @brief Enqueue multiple RD_KAFKA_OP_CONSUMER_ERR ops on \p rkq for + * a range of offsets, used for share consumer MessageSet-level errors. + * + * @param broker_id Is the relevant broker id, or RD_KAFKA_NODEID_UA (-1) + * if not applicable. + * @param err Error code. + * @param version Queue version barrier, or 0 if not applicable. + * @param rktp Toppar reference (required for share consumers). + * @param start_offset First offset in the error range (inclusive). + * @param end_offset Last offset in the error range (inclusive). + * @param ack_type Acknowledgement type for share consumer + * (REJECT for permanent errors, RELEASE for retryable). + * + * Creates one error op per offset in the range [start_offset, end_offset]. + * Each op has ack_type pre-set for share consumer acknowledgement tracking. + * + * @sa rd_kafka_consumer_err() + */ +void rd_kafka_share_msgset_err_ops( + rd_kafka_q_t *rkq, + int32_t broker_id, + rd_kafka_resp_err_t err, + int32_t version, + rd_kafka_toppar_t *rktp, + int64_t start_offset, + int64_t end_offset, + rd_kafka_share_internal_acknowledgement_type ack_type, + const char *fmt, + ...) { + va_list ap; + char buf[2048]; + int64_t offset; + + /* Format error message once for all offsets */ + va_start(ap, fmt); + rd_vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + + /* Create one error op per offset in the range */ + for (offset = start_offset; offset <= end_offset; offset++) { + rd_kafka_op_t *rko; + + rko = rd_kafka_op_new(RD_KAFKA_OP_CONSUMER_ERR); + rko->rko_version = version; + rko->rko_err = err; + rko->rko_u.err.offset = offset; + rko->rko_u.err.errstr = rd_strdup(buf); + rko->rko_u.err.rkm.rkm_broker_id = broker_id; + + /* Set acknowledgement type for share consumer */ + rko->rko_u.err.rkm.rkm_u.consumer.ack_type = ack_type; + + if (rktp) + rko->rko_rktp = rd_kafka_toppar_keep(rktp); + + rd_kafka_q_enq(rkq, rko); + } +} + + /** * Creates a reply op based on 'rko_orig'. * If 'rko_orig' has rko_op_cb set the reply op will be OR:ed with @@ -995,3 +1154,66 @@ void rd_kafka_fetch_op_app_prepare(rd_kafka_t *rk, rd_kafka_op_t *rko) { rd_kafka_update_app_pos(rk, rktp, pos, RD_DO_LOCK); } + +/** + * @brief Get offset from an op (FETCH or CONSUMER_ERR). + * + * @param rko The op to get offset from. + * @returns The offset, or RD_KAFKA_OFFSET_INVALID if not applicable. + */ +int64_t rd_kafka_op_get_offset(const rd_kafka_op_t *rko) { + if (!rko) + return RD_KAFKA_OFFSET_INVALID; + if (rko->rko_type == RD_KAFKA_OP_FETCH) + return rko->rko_u.fetch.rkm.rkm_rkmessage.offset; + if (rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) + return rko->rko_u.err.offset; + return RD_KAFKA_OFFSET_INVALID; +} + +/** + * @brief Process a share fetch response op and deliver messages. + * + * @param rko The share fetch response op + * @param rkshare The share consumer handle + * @param rkmessages Output array for messages + * @param cnt Current count of messages in array + * + * @returns Updated count of messages in array + */ +unsigned int +rd_kafka_op_process_share_fetch_response(rd_kafka_op_t *rko, + rd_kafka_share_t *rkshare, + rd_kafka_message_t **rkmessages, + unsigned int cnt) { + /* + *TODO KIP-932: send messages from the fetcher.c directly. + */ + rd_kafka_op_t *msg_rko; + int i; + int total_msgs = + rd_list_cnt(rko->rko_u.share_fetch_response.message_rkos); + + /* Build acknowledgement mapping from inflight_acks */ + rd_kafka_share_build_inflight_acks_map(rkshare, rko); + + /* Process all messages from the list. */ + for (i = 0; i < total_msgs; i++) { + msg_rko = rd_list_elem( + rko->rko_u.share_fetch_response.message_rkos, i); + + /* Return message to application */ + /** + * TODO KIP-932: We need to expose the error string maybe + * as rd_kafka_error_t in the message op. + */ + rkmessages[cnt++] = rd_kafka_message_get(msg_rko); + } + + /* Messages handed to app; clear list so op destructor + * knows not to free the message ops individually. */ + rd_list_destroy(rko->rko_u.share_fetch_response.message_rkos); + rko->rko_u.share_fetch_response.message_rkos = NULL; + + return cnt; +} diff --git a/lib/librdkafka-2.10.1/src/rdkafka_op.h b/lib/librdkafka-2.15.0/src/rdkafka_op.h similarity index 71% rename from lib/librdkafka-2.10.1/src/rdkafka_op.h rename to lib/librdkafka-2.15.0/src/rdkafka_op.h index e79309aa021..c95185932e8 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_op.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_op.h @@ -189,6 +189,37 @@ typedef enum { RD_KAFKA_OP_ELECTLEADERS, /**< Admin: * ElectLeaders * u.admin_request */ + RD_KAFKA_OP_SHARE_FETCH, /**< broker op: Issue share fetch request if + applicable. */ + RD_KAFKA_OP_SHARE_FETCH_FANOUT, /**< fanout share fetch operation */ + RD_KAFKA_OP_SHARE_COMMIT_ASYNC_FANOUT, /**< fanout share commit async + * operation (ack-only, + * no fetch) */ + RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT, /**< fanout share commit sync + * operation. App thread + * blocks until main thread + * sends response. */ + RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT_REPLY, /**< Reply from main thread + * to app thread for + * commit sync. Carries + * per-partition results. + */ + RD_KAFKA_OP_SHARE_SESSION_PARTITION_ADD, /**< share session: + * add partition */ + RD_KAFKA_OP_SHARE_SESSION_PARTITION_REMOVE, /**< share session: + * remove partition */ + RD_KAFKA_OP_SHARE_FETCH_RESPONSE, /**< Share fetch response containing + * all messages and partition acks + * from a single broker response. */ + RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE, /**< Share acknowledgement + * callback reply: main -> app + */ + RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER, /**< Register/unregister share + * acknowledgement callback: + * app -> main */ + RD_KAFKA_OP_SHARE_SESSION_CLEAR, /**< broker op: Enqueued by main thread + to clear share session during broker + decommission */ RD_KAFKA_OP__END } rd_kafka_op_type_t; @@ -580,14 +611,18 @@ struct rd_kafka_op_s { enum { RD_KAFKA_MOCK_CMD_TOPIC_SET_ERROR, RD_KAFKA_MOCK_CMD_TOPIC_CREATE, + RD_KAFKA_MOCK_CMD_TOPIC_DELETE, + RD_KAFKA_MOCK_CMD_PART_DELETE_RECORDS, RD_KAFKA_MOCK_CMD_PART_SET_LEADER, RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER, RD_KAFKA_MOCK_CMD_PART_SET_FOLLOWER_WMARKS, RD_KAFKA_MOCK_CMD_PART_PUSH_LEADER_RESPONSE, + RD_KAFKA_MOCK_CMD_PART_PUSH_REQUEST_ERRORS, RD_KAFKA_MOCK_CMD_BROKER_SET_UPDOWN, RD_KAFKA_MOCK_CMD_BROKER_SET_RTT, RD_KAFKA_MOCK_CMD_BROKER_SET_RACK, RD_KAFKA_MOCK_CMD_BROKER_DECOMMISSION, + RD_KAFKA_MOCK_CMD_BROKER_REMOVE_FROM_METADATA, RD_KAFKA_MOCK_CMD_BROKER_ADD, RD_KAFKA_MOCK_CMD_COORD_SET, RD_KAFKA_MOCK_CMD_APIVERSION_SET, @@ -595,56 +630,63 @@ struct rd_kafka_op_s { RD_KAFKA_MOCK_CMD_TELEMETRY_PUSH_INTERVAL_SET, } cmd; - rd_kafka_resp_err_t err; /**< Error for: - * TOPIC_SET_ERROR */ - char *name; /**< For: - * TOPIC_SET_ERROR - * TOPIC_CREATE - * PART_SET_FOLLOWER - * PART_SET_FOLLOWER_WMARKS - * BROKER_SET_RACK - * COORD_SET (key_type) - * PART_PUSH_LEADER_RESPONSE - */ - char *str; /**< For: - * COORD_SET (key) */ - int32_t partition; /**< For: - * PART_SET_FOLLOWER - * PART_SET_FOLLOWER_WMARKS - * PART_SET_LEADER - * APIVERSION_SET (ApiKey) - * PART_PUSH_LEADER_RESPONSE - */ - int32_t broker_id; /**< For: - * PART_SET_FOLLOWER - * PART_SET_LEADER - * BROKER_SET_UPDOWN - * BROKER_SET_RACK - * BROKER_DECOMMISSION - * BROKER_ADD - * COORD_SET */ - int64_t lo; /**< Low offset, for: - * TOPIC_CREATE (part cnt) - * PART_SET_FOLLOWER_WMARKS - * BROKER_SET_UPDOWN - * APIVERSION_SET (minver) - * BROKER_SET_RTT - */ - int64_t hi; /**< High offset, for: - * TOPIC_CREATE (repl fact) - * PART_SET_FOLLOWER_WMARKS - * APIVERSION_SET (maxver) - * REQUESTED_METRICS_SET (metrics_cnt) - * TELEMETRY_PUSH_INTERVAL_SET (interval) - */ - int32_t leader_id; /**< Leader id, for: - * PART_PUSH_LEADER_RESPONSE - */ - int32_t leader_epoch; /**< Leader epoch, for: - * PART_PUSH_LEADER_RESPONSE - */ - char **metrics; /**< Metrics requested, for: - * REQUESTED_METRICS_SET */ + rd_kafka_resp_err_t err; /**< Error for: + * TOPIC_SET_ERROR */ + char *name; /**< For: + * TOPIC_SET_ERROR + * TOPIC_CREATE + * PART_SET_FOLLOWER + * PART_SET_FOLLOWER_WMARKS + * BROKER_SET_RACK + * COORD_SET (key_type) + * PART_PUSH_LEADER_RESPONSE + */ + char *str; /**< For: + * COORD_SET (key) + */ + int32_t partition; /**< For: + * PART_SET_FOLLOWER + * PART_SET_FOLLOWER_WMARKS + * PART_SET_LEADER + * APIVERSION_SET (ApiKey) + * PART_PUSH_LEADER_RESPONSE + */ + int32_t broker_id; /**< For: + * PART_SET_FOLLOWER + * PART_SET_LEADER + * BROKER_SET_UPDOWN + * BROKER_SET_RACK + * BROKER_DECOMMISSION + * BROKER_ADD + * COORD_SET */ + int64_t lo; /**< Low offset, for: + * TOPIC_CREATE (part cnt) + * PART_SET_FOLLOWER_WMARKS + * BROKER_SET_UPDOWN + * APIVERSION_SET (minver) + * BROKER_SET_RTT + * PART_DELETE_RECORDS + * (before_offset) + */ + int64_t hi; /**< High offset, for: + * TOPIC_CREATE (repl fact) + * PART_SET_FOLLOWER_WMARKS + * APIVERSION_SET (maxver) + * REQUESTED_METRICS_SET (metrics_cnt) + * TELEMETRY_PUSH_INTERVAL_SET (interval) + */ + int32_t leader_id; /**< Leader id, for: + * PART_PUSH_LEADER_RESPONSE + */ + int32_t leader_epoch; /**< Leader epoch, for: + * PART_PUSH_LEADER_RESPONSE + */ + char **metrics; /**< Metrics requested, for: + * REQUESTED_METRICS_SET */ + rd_kafka_resp_err_t *errs; /**< Errors to push, for: + * PART_PUSH_REQUEST_ERRORS + * (ApiKey in .lo, + * count in .hi) */ } mock; struct { @@ -722,8 +764,155 @@ struct rd_kafka_op_s { /** Termination callback to trigger * on the op handler's thread. */ void (*cb)(rd_kafka_t *rk, void *rkb); + + /** Share-consumer final pending-ack batches carried + * on the cgrp TERMINATE op. NULL for non-share + * consumers and once the list has been dispatched. + * Type: rd_kafka_share_ack_batches_t* (rd_list_t). + * Freed by the op destructor if still set. */ + rd_list_t *ack_batches; } terminated; + struct { + + rd_bool_t should_leave; /**< Whether this broker should + * leave the share-fetch + * session. */ + + /** Whether this broker should share-fetch nonzero + * messages. */ + rd_bool_t should_fetch; + + /** Absolute timeout left to complete this share-fetch. + */ + rd_ts_t abs_timeout; + + /** Target broker to which op is sent. */ + rd_kafka_broker_t *target_broker; + + /** Whether records were fetched in this + * share-fetch response. Set by broker + * thread, read by main thread in reply + * handler. */ + rd_bool_t records_fetched; + + /** Ack batches to send with this request. + * Type: rd_kafka_share_ack_batches_t*. + * Moved from rkb_share_async_ack_details + * when creating the op. Freed by broker + * thread after use. + * TODO KIP-932: Change name. + */ + rd_list_t *ack_details; + + /** commit_sync request ID that this op belongs + * to, or 0 if not a commit_sync op. Compared + * with rkcg_commit_sync_request.id to detect + * stale responses. */ + int64_t commit_sync_request_id; + } share_fetch; + + struct { + /** Whether this FANOUT should fetch more records. + * When rd_true, the selected broker's SHARE_FETCH + * op will have should_fetch=true. + * When rd_false, this is an ack-only FANOUT and + * no broker will fetch new records. */ + rd_bool_t fetch_more_records; + + /** List of all acknowledgement batches to send. + * Type: rd_kafka_share_ack_batches_t* + * Built from inflight ack map, will be filtered + * by leader when creating SHARE_FETCH ops. + * Each entry uses size=1 with types[0] holding the + * single ack type for the collated range. + * Set to NULL after ownership is transferred + * to per-broker ack_details. + * TODO KIP-932: Change name + */ + rd_list_t *ack_batches; + } share_fetch_fanout; + + struct { + /** List of all acknowledgement batches to commit. + * Type: rd_kafka_share_ack_batches_t* + * Built from inflight ack map, will be segregated + * by leader and sent to respective brokers. + * Set to NULL after ownership is transferred + * to per-broker ack_details. */ + rd_list_t *ack_batches; + } share_commit_async_fanout; + + struct { + /** List of all acknowledgement batches to commit. + * Type: rd_kafka_share_ack_batches_t* + * Built from inflight ack map, will be segregated + * by leader and sent to respective brokers. + * Set to NULL after ownership is transferred + * to per-broker ack_details. */ + rd_list_t *ack_batches; + + /** Absolute timeout for the commit_sync request. + * Calculated from user-provided timeout_ms. */ + rd_ts_t abs_timeout; + + /** Per-partition results (topic, partition, err). + * Set in response op by main thread. + * Read by app thread after unblocking. */ + rd_kafka_topic_partition_list_t *results; + } share_commit_sync_fanout; + + struct { + /** Per-partition results (topic, partition, err). + * Moved from commit_sync_request by main thread. */ + rd_kafka_topic_partition_list_t *results; + } share_commit_sync_fanout_reply; + + /** + * Share fetch response - single rko containing all messages + * and partition ack info from one broker response. + */ + struct { + /** List of message ops (rd_kafka_op_t*). + * Contains only actual messages (ACQUIRED/REJECT), + * no GAP placeholder ops. + */ + /* + * TODO KIP-932: Check if we can send the messages only + * instead of the message rkos. + */ + rd_list_t *message_rkos; + + /** List of per-partition inflight ack mappings. + * Type: rd_kafka_share_ack_batches_t* + * Contains per-offset ack types (ACQUIRED/GAP/REJECT). + * Built in broker thread, merged to rkshare in app + * thread. + */ + rd_list_t *inflight_acks; + } share_fetch_response; + + /** + * Share acknowledgement callback reply. + * Contains results to deliver to the runtime acknowledgement + * callback set via + * rd_kafka_share_set_acknowledgement_commit_cb(). + */ + struct { + /** List of partition offsets. */ + rd_kafka_share_partition_offsets_list_t *partitions; + /* Callback is looked up from rk->rk_rkshare at + * invoke time. */ + } share_ack_commit_cb_execute; + + /** Share acknowledgement callback registration op. + * Sent from app thread to main thread when the runtime + * callback registration state transitions (set ↔ unset). */ + struct { + rd_bool_t + registered; /**< true=register, false=unregister */ + } share_ack_commit_cb_register; + } rko_u; }; @@ -774,6 +963,17 @@ void rd_kafka_consumer_err(rd_kafka_q_t *rkq, int64_t offset, const char *fmt, ...) RD_FORMAT(printf, 8, 9); +void rd_kafka_share_msgset_err_ops( + rd_kafka_q_t *rkq, + int32_t broker_id, + rd_kafka_resp_err_t err, + int32_t version, + rd_kafka_toppar_t *rktp, + int64_t start_offset, + int64_t end_offset, + rd_kafka_share_internal_acknowledgement_type ack_type, + const char *fmt, + ...) RD_FORMAT(printf, 9, 10); rd_kafka_op_t *rd_kafka_op_req0(rd_kafka_q_t *destq, rd_kafka_q_t *recvq, rd_kafka_op_t *rko, @@ -823,6 +1023,13 @@ void rd_kafka_op_print(FILE *fp, const char *prefix, rd_kafka_op_t *rko); void rd_kafka_fetch_op_app_prepare(rd_kafka_t *rk, rd_kafka_op_t *rko); +int64_t rd_kafka_op_get_offset(const rd_kafka_op_t *rko); + +unsigned int +rd_kafka_op_process_share_fetch_response(rd_kafka_op_t *rko, + rd_kafka_share_t *rkshare, + rd_kafka_message_t **rkmessages, + unsigned int cnt); #define rd_kafka_op_is_ctrl_msg(rko) \ ((rko)->rko_type == RD_KAFKA_OP_FETCH && !(rko)->rko_err && \ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_partition.c b/lib/librdkafka-2.15.0/src/rdkafka_partition.c similarity index 94% rename from lib/librdkafka-2.10.1/src/rdkafka_partition.c rename to lib/librdkafka-2.15.0/src/rdkafka_partition.c index a88d77a489b..f7e5906a4a0 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_partition.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_partition.c @@ -146,8 +146,8 @@ static void rd_kafka_toppar_consumer_lag_req(rd_kafka_toppar_t *rktp) { * broker supports FETCH >= v5, since this will be set when * doing fetch requests. */ - if (rd_kafka_broker_ApiVersion_supported( - rktp->rktp_broker, RD_KAFKAP_Fetch, 0, 5, NULL) == 5) { + if (rd_kafka_broker_ApiVersion_at_least(rktp->rktp_broker, + RD_KAFKAP_Fetch, 5)) { rd_kafka_toppar_unlock(rktp); return; } @@ -198,6 +198,9 @@ static void rd_kafka_toppar_consumer_lag_tmr_cb(rd_kafka_timers_t *rkts, void rd_kafka_toppar_op_version_bump(rd_kafka_toppar_t *rktp, int32_t version) { rd_kafka_op_t *rko; + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) + return; + rktp->rktp_op_version = version; rko = rd_kafka_op_new(RD_KAFKA_OP_BARRIER); rko->rko_version = version; @@ -260,7 +263,11 @@ rd_kafka_toppar_t *rd_kafka_toppar_new0(rd_kafka_topic_t *rkt, rktp->rktp_ops = rd_kafka_q_new(rkt->rkt_rk); rktp->rktp_ops->rkq_serve = rd_kafka_toppar_op_serve; rktp->rktp_ops->rkq_opaque = rktp; - rd_atomic32_init(&rktp->rktp_version, 1); + if (RD_KAFKA_IS_SHARE_CONSUMER(rkt->rkt_rk)) { + rd_atomic32_init(&rktp->rktp_version, 0); + } else { + rd_atomic32_init(&rktp->rktp_version, 1); + } rktp->rktp_op_version = rd_atomic32_get(&rktp->rktp_version); rd_atomic32_init(&rktp->rktp_msgs_inflight, 0); @@ -474,6 +481,65 @@ rd_kafka_toppar_t *rd_kafka_toppar_get2(rd_kafka_t *rk, } +/** + * @brief Same as rd_kafka_toppar_get2() but looks up (or creates) the + * topic by topic_id rather than by name. + * + * Used by share-consumer paths where the heartbeat-derived + * assignment is the source of truth and the topic id is the + * canonical identity. Looking up by id avoids matching a + * recreated topic's response to a different rkt that happens + * to share its name. + * + * topic_id must be non-zero; topic name may be NULL. + * + * @locality any + * @locks none + */ +rd_kafka_toppar_t *rd_kafka_toppar_get_by_id2(rd_kafka_t *rk, + const char *topic, + rd_kafka_Uuid_t topic_id, + int32_t partition, + int create_on_miss) { + rd_kafka_topic_t *rkt; + rd_kafka_toppar_t *rktp; + + rd_dassert(!RD_KAFKA_UUID_IS_ZERO(topic_id)); + + rd_kafka_wrlock(rk); + + rkt = rd_kafka_topic_find_by_topic_id(rk, topic_id); + if (unlikely(!rkt)) { + if (!create_on_miss) { + rd_kafka_wrunlock(rk); + return NULL; + } + rkt = rd_kafka_topic_new_with_id(rk, topic, topic_id, + 0 /*no-lock*/); + if (!rkt) { + rd_kafka_wrunlock(rk); + rd_kafka_log(rk, LOG_ERR, "TOPIC", + "Failed to create local topic \"%s\" " + "(id %s): %s", + topic ? topic : "(null)", + rd_kafka_Uuid_base64str(&topic_id), + rd_strerror(errno)); + return NULL; + } + } + + rd_kafka_wrunlock(rk); + + rd_kafka_topic_wrlock(rkt); + rktp = rd_kafka_toppar_desired_add(rkt, partition); + rd_kafka_topic_wrunlock(rkt); + + rd_kafka_topic_destroy0(rkt); + + return rktp; +} + + /** * Returns a toppar if it is available in the cluster. * '*errp' is set to the error-code if lookup fails. @@ -1131,8 +1197,13 @@ static void rd_kafka_toppar_broker_migrate(rd_kafka_toppar_t *rktp, * prior to leaving the broker to avoid stalling * on the new broker waiting for a offset reply from * this old broker (that might not come and thus need - * to time out..slowly) */ - if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT) + * to time out..slowly). + * + * Share consumers never enter OFFSET_WAIT so this branch is + * unreachable for them today; guard regardless so the offset + * machinery stays out of the share leader-change path. */ + if (!RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk) && + rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT) rd_kafka_toppar_offset_retry(rktp, 500, "migrating to new broker"); @@ -1196,7 +1267,8 @@ void rd_kafka_toppar_broker_leave_for_remove(rd_kafka_toppar_t *rktp) { * on the new broker waiting for a offset reply from * this old broker (that might not come and thus need * to time out..slowly) */ - if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT) + if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT && + !RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) rd_kafka_toppar_set_fetch_state( rktp, RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY); @@ -1575,6 +1647,13 @@ static void rd_kafka_toppar_offset_retry(rd_kafka_toppar_t *rktp, rd_ts_t tmr_next; int restart_tmr; + /* Share consumers never use the offset-query state machine. + * Callers must guard; this early-return is belt-and-suspenders + * so a stray future call doesn't arm the offset-query timer + * for a share-assigned rktp. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) + return; + /* (Re)start timer if not started or the current timeout * is larger than \p backoff_ms. */ tmr_next = rd_kafka_timer_next(&rktp->rktp_rkt->rkt_rk->rk_timers, @@ -2085,6 +2164,8 @@ static rd_kafka_op_res_t rd_kafka_toppar_op_serve(rd_kafka_t *rk, rd_kafka_toppar_t *rktp = NULL; int outdated = 0; + rd_dassert(!RD_KAFKA_IS_SHARE_CONSUMER(rk)); + if (rko->rko_rktp) rktp = rko->rko_rktp; @@ -2162,6 +2243,7 @@ static rd_kafka_op_res_t rd_kafka_toppar_op_serve(rd_kafka_t *rk, rd_kafka_toppar_lock(rktp); if (rko->rko_err) { + int actions; rd_kafka_dbg( rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET", "Failed to fetch offset for " @@ -2176,10 +2258,15 @@ static rd_kafka_op_res_t rd_kafka_toppar_op_serve(rd_kafka_t *rk, rd_kafka_toppar_unlock(rktp); - /* Propagate error to application */ + actions = rd_kafka_handle_OffsetFetch_err_action( + NULL, rko->rko_err, NULL); + /* Propagate error to application. Exclude + * permanent errors that caused a coordinator + * refresh like `NOT_COORDINATOR` */ if (rko->rko_err != RD_KAFKA_RESP_ERR__WAIT_COORD && rko->rko_err != - RD_KAFKA_RESP_ERR_UNSTABLE_OFFSET_COMMIT) + RD_KAFKA_RESP_ERR_UNSTABLE_OFFSET_COMMIT && + !(actions & RD_KAFKA_ERR_ACTION_REFRESH)) rd_kafka_consumer_err( rktp->rktp_fetchq, RD_KAFKA_NODEID_UA, rko->rko_err, 0, NULL, rktp, @@ -2283,6 +2370,16 @@ rd_kafka_resp_err_t rd_kafka_toppar_op_fetch_start(rd_kafka_toppar_t *rktp, rd_kafka_replyq_t replyq) { int32_t version; + rd_dassert(!RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)); + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "CONSUMER", + "consume_start is not supported for share " + "consumers: %s [%" PRId32 "]: ignored", + rd_kafka_topic_name_safe(rktp->rktp_rkt), + rktp->rktp_partition); + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + rd_kafka_q_lock(rktp->rktp_fetchq); if (fwdq && !(rktp->rktp_fetchq->rkq_flags & RD_KAFKA_Q_F_FWD_APP)) rd_kafka_q_fwd_set0(rktp->rktp_fetchq, fwdq, 0, /* no do_lock */ @@ -2315,6 +2412,16 @@ rd_kafka_resp_err_t rd_kafka_toppar_op_fetch_stop(rd_kafka_toppar_t *rktp, rd_kafka_replyq_t replyq) { int32_t version; + rd_dassert(!RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)); + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "CONSUMER", + "consume_stop is not supported for share " + "consumers: %s [%" PRId32 "]: ignored", + rd_kafka_topic_name_safe(rktp->rktp_rkt), + rktp->rktp_partition); + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + /* Bump version barrier. */ version = rd_kafka_toppar_version_new_barrier(rktp); @@ -2344,6 +2451,16 @@ rd_kafka_resp_err_t rd_kafka_toppar_op_seek(rd_kafka_toppar_t *rktp, rd_kafka_replyq_t replyq) { int32_t version; + rd_dassert(!RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)); + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "CONSUMER", + "seek is not supported for share consumers: " + "%s [%" PRId32 "]: ignored", + rd_kafka_topic_name_safe(rktp->rktp_rkt), + rktp->rktp_partition); + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + /* Bump version barrier. */ version = rd_kafka_toppar_version_new_barrier(rktp); @@ -2373,7 +2490,20 @@ rd_kafka_resp_err_t rd_kafka_toppar_op_pause_resume(rd_kafka_toppar_t *rktp, int flag, rd_kafka_replyq_t replyq) { int32_t version; - rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_PAUSE); + rd_kafka_op_t *rko; + + rd_dassert(!RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)); + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, + pause ? "PAUSE" : "RESUME", + "Pause/resume is not supported for share " + "consumers: %s [%" PRId32 "]: ignored", + rd_kafka_topic_name_safe(rktp->rktp_rkt), + rktp->rktp_partition); + return RD_KAFKA_RESP_ERR_NO_ERROR; + } + + rko = rd_kafka_op_new(RD_KAFKA_OP_PAUSE); if (!pause) { /* If partitions isn't paused, avoid bumping its version, @@ -2511,6 +2641,39 @@ rd_kafka_toppars_pause_resume(rd_kafka_t *rk, } +/** + * @brief Share-consumer variant of rd_kafka_toppar_enq_error. + * + * Records the (topic, err) on the cgrp's per-cycle accumulator + * for delivery or logging at the end of the metadata cycle. + * Multiple calls for the same topic in one cycle collapse to + * a single entry. + * + * @locality rdkafka main thread + */ +static void rd_kafka_share_toppar_enq_error(rd_kafka_toppar_t *rktp, + rd_kafka_resp_err_t err) { + rd_kafka_cgrp_t *rkcg = rktp->rktp_rkt->rkt_rk->rk_cgrp; + rd_kafka_topic_t *rkt = rktp->rktp_rkt; + const char *topic = rkt->rkt_topic->str; + rd_kafka_Uuid_t topic_id = rkt->rkt_topic_id; + rd_kafka_topic_partition_t *prev; + + if (!rkcg || !rkcg->rkcg_errored_topics) + return; + + prev = rd_kafka_topic_partition_list_find_topic_by_id( + rkcg->rkcg_errored_topics, topic_id); + if (prev) { + prev->err = err; + } else { + rd_kafka_topic_partition_list_add_with_topic_name_and_id( + rkcg->rkcg_errored_topics, topic_id, topic, + RD_KAFKA_PARTITION_UA) + ->err = err; + } +} + /** * Propagate error for toppar @@ -2521,6 +2684,11 @@ void rd_kafka_toppar_enq_error(rd_kafka_toppar_t *rktp, rd_kafka_op_t *rko; char buf[512]; + if (RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { + rd_kafka_share_toppar_enq_error(rktp, err); + return; + } + rko = rd_kafka_op_new(RD_KAFKA_OP_ERR); rko->rko_err = err; rko->rko_rktp = rd_kafka_toppar_keep(rktp); @@ -2588,6 +2756,32 @@ void rd_kafka_toppar_leader_unavailable(rd_kafka_toppar_t *rktp, rd_false /* don't force */); } +/** + * @locality any + */ +rd_bool_t rd_kafka_toppar_is_on_cgrp(rd_kafka_toppar_t *rktp, + rd_bool_t do_lock) { + rd_bool_t on_cgrp; + if (do_lock) { + rd_kafka_toppar_lock(rktp); + } + on_cgrp = + (rktp->rktp_flags & RD_KAFKA_TOPPAR_F_ON_CGRP) ? rd_true : rd_false; + + if (do_lock) { + rd_kafka_toppar_unlock(rktp); + } + + return on_cgrp; +} + +/** + * @brief Toppar copier for rd_list_copy() + */ +void *rd_kafka_toppar_list_copy(const void *elem, void *opaque) { + return rd_kafka_toppar_keep((rd_kafka_toppar_t *)elem); +} + const char * rd_kafka_topic_partition_topic(const rd_kafka_topic_partition_t *rktpar) { @@ -2679,6 +2873,16 @@ rd_kafka_topic_partition_t *rd_kafka_topic_partition_new(const char *topic, return rktpar; } +rd_kafka_topic_partition_t * +rd_kafka_topic_partition_new_with_id_and_name(rd_kafka_Uuid_t topic_id, + const char *topic, + int32_t partition) { + rd_kafka_topic_partition_t *rktpar = + rd_kafka_topic_partition_new(topic, partition); + rd_kafka_topic_partition_set_topic_id(rktpar, topic_id); + return rktpar; +} + /** * @brief Update \p dst with info from \p src. */ @@ -3166,10 +3370,24 @@ rd_kafka_topic_partition_ensure_toppar(rd_kafka_t *rk, parpriv = rd_kafka_topic_partition_get_private(rktpar); - if (!parpriv->rktp) + if (parpriv->rktp) + return parpriv->rktp; + + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + /* Share consumer identifies topics by id; looking up + * the topic by name risks matching a stale rkt that + * a recreated topic shares its name with. */ + rd_dassert(!RD_KAFKA_UUID_IS_ZERO(parpriv->topic_id)); + if (RD_KAFKA_UUID_IS_ZERO(parpriv->topic_id)) + return NULL; + parpriv->rktp = rd_kafka_toppar_get_by_id2( + rk, rktpar->topic, parpriv->topic_id, rktpar->partition, + create_on_miss); + } else { parpriv->rktp = rd_kafka_toppar_get2( rk, rktpar->topic, rktpar->partition, 0 /* not ua on miss */, create_on_miss); + } return parpriv->rktp; } @@ -4263,9 +4481,10 @@ const char *rd_kafka_topic_partition_list_str( "]" "%s" "%s", - of == 0 ? "" : ", ", rktpar->topic, - topic_id_str, rktpar->partition, offsetstr, - errstr); + of == 0 ? "" : ", ", + rd_kafka_topic_name_str_safe(rktpar->topic), + topic_id_str ? topic_id_str : "(null)", + rktpar->partition, offsetstr, errstr); if ((size_t)r >= dest_size - of) { rd_snprintf(&dest[dest_size - 4], 4, "..."); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_partition.h b/lib/librdkafka-2.15.0/src/rdkafka_partition.h similarity index 96% rename from lib/librdkafka-2.10.1/src/rdkafka_partition.h rename to lib/librdkafka-2.15.0/src/rdkafka_partition.h index 97a704f03b9..938ef839f96 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_partition.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_partition.h @@ -32,6 +32,7 @@ #include "rdkafka_topic.h" #include "rdkafka_cgrp.h" #include "rdkafka_broker.h" +#include "rdkafka_share_acknowledgement.h" extern const char *rd_kafka_fetch_states[]; @@ -283,6 +284,12 @@ struct rd_kafka_toppar_s { /* rd_kafka_toppar_t */ * Broker thread: Recv IO FetchResponse with tver=2 which * is same as rktp_version so message is forwarded * to app. + * + * Share consumers: all three versions remain 0 for the toppar's + * lifetime. rktp_version is initialised to 0 and + * rd_kafka_toppar_op_version_bump() short-circuits, so no op ever + * carries a non-zero rko_version. Any code that bumps rktp_version + * directly MUST guard on RD_KAFKA_IS_SHARE_CONSUMER. */ rd_atomic32_t rktp_version; /* Latest op version. * Authoritative (app thread)*/ @@ -533,10 +540,22 @@ void rd_kafka_toppar_destroy_final(rd_kafka_toppar_t *rktp); #define rd_kafka_toppar_destroy(RKTP) \ do { \ rd_kafka_toppar_t *_RKTP = (RKTP); \ - if (unlikely(rd_refcnt_sub(&_RKTP->rktp_refcnt) == 0)) \ - rd_kafka_toppar_destroy_final(_RKTP); \ + rd_kafka_toppar_destroy0(__FUNCTION__, __LINE__, _RKTP); \ } while (0) +/* Common destroy helper used by both the macro and the free-wrapper. */ +static RD_UNUSED RD_INLINE void +rd_kafka_toppar_destroy0(const char *func, int line, rd_kafka_toppar_t *rktp) { + if (unlikely(rd_refcnt_sub_fl(func, line, &rktp->rktp_refcnt) == 0)) + rd_kafka_toppar_destroy_final(rktp); +} + +/* Free-function compatible wrapper for rd_list_new and similar APIs + * (signature: void (*)(void *)). */ +static RD_UNUSED RD_INLINE void rd_kafka_toppar_destroy_free(void *ptr) { + rd_kafka_toppar_destroy0(__FUNCTION__, __LINE__, + (rd_kafka_toppar_t *)ptr); +} #define rd_kafka_toppar_lock(rktp) mtx_lock(&(rktp)->rktp_lock) @@ -599,6 +618,11 @@ rd_kafka_toppar_t *rd_kafka_toppar_get2(rd_kafka_t *rk, int32_t partition, int ua_on_miss, int create_on_miss); +rd_kafka_toppar_t *rd_kafka_toppar_get_by_id2(rd_kafka_t *rk, + const char *topic, + rd_kafka_Uuid_t topic_id, + int32_t partition, + int create_on_miss); rd_kafka_toppar_t *rd_kafka_toppar_get_avail(const rd_kafka_topic_t *rkt, int32_t partition, int ua_on_miss, @@ -677,9 +701,17 @@ rd_kafka_toppars_pause_resume(rd_kafka_t *rk, int flag, rd_kafka_topic_partition_list_t *partitions); +rd_bool_t rd_kafka_toppar_is_on_cgrp(rd_kafka_toppar_t *rktp, + rd_bool_t do_lock); +void *rd_kafka_toppar_list_copy(const void *elem, void *opaque); + rd_kafka_topic_partition_t *rd_kafka_topic_partition_new(const char *topic, int32_t partition); +rd_kafka_topic_partition_t * +rd_kafka_topic_partition_new_with_id_and_name(rd_kafka_Uuid_t topic_id, + const char *topic, + int32_t partition); void rd_kafka_topic_partition_destroy_free(void *ptr); rd_kafka_topic_partition_t * rd_kafka_topic_partition_copy(const rd_kafka_topic_partition_t *src); @@ -767,6 +799,7 @@ int rd_kafka_topic_partition_match(rd_kafka_t *rk, int rd_kafka_topic_partition_cmp(const void *_a, const void *_b); int rd_kafka_topic_partition_by_id_cmp(const void *_a, const void *_b); unsigned int rd_kafka_topic_partition_hash(const void *a); +unsigned int rd_kafka_topic_partition_hash_by_id(const void *a); int rd_kafka_topic_partition_list_find_idx( const rd_kafka_topic_partition_list_t *rktparlist, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_pattern.c b/lib/librdkafka-2.15.0/src/rdkafka_pattern.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_pattern.c rename to lib/librdkafka-2.15.0/src/rdkafka_pattern.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_pattern.h b/lib/librdkafka-2.15.0/src/rdkafka_pattern.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_pattern.h rename to lib/librdkafka-2.15.0/src/rdkafka_pattern.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_plugin.c b/lib/librdkafka-2.15.0/src/rdkafka_plugin.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_plugin.c rename to lib/librdkafka-2.15.0/src/rdkafka_plugin.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_plugin.h b/lib/librdkafka-2.15.0/src/rdkafka_plugin.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_plugin.h rename to lib/librdkafka-2.15.0/src/rdkafka_plugin.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_proto.h b/lib/librdkafka-2.15.0/src/rdkafka_proto.h similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_proto.h rename to lib/librdkafka-2.15.0/src/rdkafka_proto.h index f58c4e1dc97..0ad21e06fcc 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_proto.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_proto.h @@ -175,6 +175,9 @@ static RD_UNUSED const char *rd_kafka_ApiKey2str(int16_t ApiKey) { [RD_KAFKAP_ConsumerGroupDescribe] = "ConsumerGroupDescribe", [RD_KAFKAP_GetTelemetrySubscriptions] = "GetTelemetrySubscriptions", [RD_KAFKAP_PushTelemetry] = "PushTelemetry", + [RD_KAFKAP_ShareGroupHeartbeat] = "ShareGroupHeartbeat", + [RD_KAFKAP_ShareFetch] = "ShareFetch", + [RD_KAFKAP_ShareAcknowledge] = "ShareAcknowledge", }; static RD_TLS char ret[64]; @@ -623,7 +626,7 @@ static RD_INLINE RD_UNUSED int rd_kafka_Uuid_ptr_cmp(void *a, void *b) { rd_kafka_Uuid_t rd_kafka_Uuid_random(); -const char *rd_kafka_Uuid_str(const rd_kafka_Uuid_t *uuid); +char *rd_kafka_Uuid_str(const rd_kafka_Uuid_t *uuid); unsigned int rd_kafka_Uuid_hash(const rd_kafka_Uuid_t *uuid); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_protocol.h b/lib/librdkafka-2.15.0/src/rdkafka_protocol.h similarity index 94% rename from lib/librdkafka-2.10.1/src/rdkafka_protocol.h rename to lib/librdkafka-2.15.0/src/rdkafka_protocol.h index 4755494d0b0..ddbe04a9b77 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_protocol.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_protocol.h @@ -120,8 +120,12 @@ #define RD_KAFKAP_GetTelemetrySubscriptions 71 #define RD_KAFKAP_PushTelemetry 72 #define RD_KAFKAP_AssignReplicasToDirs 73 +#define RD_KAFKAP_ShareGroupHeartbeat 76 +#define RD_KAFKAP_ShareGroupDescribe 77 +#define RD_KAFKAP_ShareFetch 78 +#define RD_KAFKAP_ShareAcknowledge 79 +#define RD_KAFKAP__NUM 80 -#define RD_KAFKAP__NUM 74 - +#define RD_KAFKAP_RPC_VERSION_MAX INT16_MAX #endif /* _RDKAFKA_PROTOCOL_H_ */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_queue.c b/lib/librdkafka-2.15.0/src/rdkafka_queue.c similarity index 76% rename from lib/librdkafka-2.10.1/src/rdkafka_queue.c rename to lib/librdkafka-2.15.0/src/rdkafka_queue.c index 5ced86971dd..fb11fe08834 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_queue.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_queue.c @@ -93,10 +93,12 @@ void rd_kafka_q_init0(rd_kafka_q_t *rkq, rkq->rkq_flags = RD_KAFKA_Q_F_READY; if (for_consume) rkq->rkq_flags |= RD_KAFKA_Q_F_CONSUMER; - rkq->rkq_rk = rk; - rkq->rkq_qio = NULL; - rkq->rkq_serve = NULL; - rkq->rkq_opaque = NULL; + rkq->rkq_rk = rk; + rkq->rkq_qio = NULL; + rkq->rkq_serve = NULL; + rkq->rkq_opaque = NULL; + rkq->rkq_ts_last_poll_start = 0; + rkq->rkq_ts_last_poll_end = 0; mtx_init(&rkq->rkq_lock, mtx_plain); cnd_init(&rkq->rkq_cond); #if ENABLE_DEVEL @@ -380,16 +382,26 @@ rd_kafka_op_filter(rd_kafka_q_t *rkq, rd_kafka_op_t *rko, int version) { * Serve q like rd_kafka_q_serve() until an op is found that can be returned * as an event to the application. * + * @param rkq Queue to pop from. + * @param timeout_us Maximum time to wait for an op, in microseconds. + * @param version Fetch version to filter out outdated ops. + * @param cb_type Callback type to use for the op. + * @param callback Callback to use for the op, if any. + * @param opaque Opaque pointer to pass to the callback. + * @param is_consume_call If `rd_true` and it could be a consumer call it + * checks if this queue can contain fetched messages. + * * @returns the first event:able op, or NULL on timeout. * - * Locality: any thread + * @locality any thread */ -rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, - rd_ts_t timeout_us, - int32_t version, - rd_kafka_q_cb_type_t cb_type, - rd_kafka_q_serve_cb_t *callback, - void *opaque) { +static rd_kafka_op_t *rd_kafka_q_pop_serve0(rd_kafka_q_t *rkq, + rd_ts_t timeout_us, + int32_t version, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque, + rd_bool_t is_consume_call) { rd_kafka_op_t *rko; rd_kafka_q_t *fwdq; @@ -400,12 +412,14 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, rd_kafka_yield_thread = 0; if (!(fwdq = rd_kafka_q_fwd_get(rkq, 0))) { const rd_bool_t can_q_contain_fetched_msgs = + is_consume_call && rd_kafka_q_can_contain_fetched_msgs(rkq, RD_DONT_LOCK); rd_ts_t abs_timeout = rd_timeout_init_us(timeout_us); if (can_q_contain_fetched_msgs) - rd_kafka_app_poll_start(rkq->rkq_rk, 0, timeout_us); + rd_kafka_app_poll_start(rkq->rkq_rk, rkq, 0, + timeout_us); while (1) { rd_kafka_op_res_t res; @@ -445,14 +459,14 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, } else if (unlikely(res == RD_KAFKA_OP_RES_YIELD)) { if (can_q_contain_fetched_msgs) - rd_kafka_app_polled( - rkq->rkq_rk); + rd_kafka_app_polled(rkq->rkq_rk, + rkq); /* Callback yielded, unroll */ return NULL; } else { if (can_q_contain_fetched_msgs) - rd_kafka_app_polled( - rkq->rkq_rk); + rd_kafka_app_polled(rkq->rkq_rk, + rkq); break; /* Proper op, handle below. */ } } @@ -461,7 +475,7 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, if (is_locked) mtx_unlock(&rkq->rkq_lock); if (can_q_contain_fetched_msgs) - rd_kafka_app_polled(rkq->rkq_rk); + rd_kafka_app_polled(rkq->rkq_rk, rkq); return NULL; } @@ -472,7 +486,7 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, abs_timeout) != thrd_success) { mtx_unlock(&rkq->rkq_lock); if (can_q_contain_fetched_msgs) - rd_kafka_app_polled(rkq->rkq_rk); + rd_kafka_app_polled(rkq->rkq_rk, rkq); return NULL; } } @@ -481,8 +495,8 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, /* Since the q_pop may block we need to release the parent * queue's lock. */ mtx_unlock(&rkq->rkq_lock); - rko = rd_kafka_q_pop_serve(fwdq, timeout_us, version, cb_type, - callback, opaque); + rko = rd_kafka_q_pop_serve0(fwdq, timeout_us, version, cb_type, + callback, opaque, is_consume_call); rd_kafka_q_destroy(fwdq); } @@ -490,28 +504,64 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, return rko; } +rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, + rd_ts_t timeout_us, + int32_t version, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque) { + return rd_kafka_q_pop_serve0(rkq, timeout_us, version, cb_type, + callback, opaque, rd_false); +} + +/** + * @brief Same as `rd_kafka_q_pop_serve`, use this call when the queue + * could be a fetch queue, use the other one when it + * can never be. + */ +rd_kafka_op_t * +rd_kafka_q_pop_serve_maybe_consume(rd_kafka_q_t *rkq, + rd_ts_t timeout_us, + int32_t version, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque) { + return rd_kafka_q_pop_serve0(rkq, timeout_us, version, cb_type, + callback, opaque, + /* Only check if to call app_polled when + * this is a consumer. */ + rkq->rkq_rk->rk_type == RD_KAFKA_CONSUMER); +} + rd_kafka_op_t * rd_kafka_q_pop(rd_kafka_q_t *rkq, rd_ts_t timeout_us, int32_t version) { return rd_kafka_q_pop_serve(rkq, timeout_us, version, RD_KAFKA_Q_CB_RETURN, NULL, NULL); } - /** * Pop all available ops from a queue and call the provided * callback for each op. - * `max_cnt` limits the number of ops served, 0 = no limit. * - * Returns the number of ops served. + * @param rkq Queue to serve. + * @param max_cnt Limits the number of ops served, 0 = no limit. + * @param cb_type Callback type to use. + * @param callback Callback to call for each op. + * @param opaque Opaque pointer to pass to the callback. + * @param is_consume_call If `rd_true` and it could be a consumer call it + * checks if this queue can contain fetched messages. * - * Locality: any thread. + * @return The number of ops served. + * + * @locality any thread. */ -int rd_kafka_q_serve(rd_kafka_q_t *rkq, - int timeout_ms, - int max_cnt, - rd_kafka_q_cb_type_t cb_type, - rd_kafka_q_serve_cb_t *callback, - void *opaque) { +int rd_kafka_q_serve0(rd_kafka_q_t *rkq, + int timeout_ms, + int max_cnt, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque, + rd_bool_t is_consume_call) { rd_kafka_t *rk = rkq->rkq_rk; rd_kafka_op_t *rko; rd_kafka_q_t localq; @@ -519,6 +569,7 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq, int cnt = 0; rd_ts_t abs_timeout; const rd_bool_t can_q_contain_fetched_msgs = + is_consume_call && rd_kafka_q_can_contain_fetched_msgs(rkq, RD_DONT_LOCK); rd_dassert(cb_type); @@ -531,8 +582,8 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq, /* Since the q_pop may block we need to release the parent * queue's lock. */ mtx_unlock(&rkq->rkq_lock); - ret = rd_kafka_q_serve(fwdq, timeout_ms, max_cnt, cb_type, - callback, opaque); + ret = rd_kafka_q_serve0(fwdq, timeout_ms, max_cnt, cb_type, + callback, opaque, is_consume_call); rd_kafka_q_destroy(fwdq); return ret; } @@ -541,7 +592,7 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq, abs_timeout = rd_timeout_init(timeout_ms); if (can_q_contain_fetched_msgs) - rd_kafka_app_poll_start(rk, 0, timeout_ms); + rd_kafka_app_poll_start(rk, rkq, 0, timeout_ms); /* Wait for op */ while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) && @@ -555,7 +606,7 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq, if (!rko) { mtx_unlock(&rkq->rkq_lock); if (can_q_contain_fetched_msgs) - rd_kafka_app_polled(rk); + rd_kafka_app_polled(rk, rkq); return 0; } @@ -591,13 +642,41 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq, } if (can_q_contain_fetched_msgs) - rd_kafka_app_polled(rk); + rd_kafka_app_polled(rk, rkq); rd_kafka_q_destroy_owner(&localq); return cnt; } +int rd_kafka_q_serve(rd_kafka_q_t *rkq, + int timeout_ms, + int max_cnt, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque) { + return rd_kafka_q_serve0(rkq, timeout_ms, max_cnt, cb_type, callback, + opaque, rd_false); +} + +/** + * @brief Same as `rd_kafka_q_serve`, use this call when the queue + * could be a fetch queue, use the other one when it + * can never be. + */ +int rd_kafka_q_serve_maybe_consume(rd_kafka_q_t *rkq, + int timeout_ms, + int max_cnt, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque) { + return rd_kafka_q_serve0(rkq, timeout_ms, max_cnt, cb_type, callback, + opaque, + /* Only check if to call app_polled when + * this is a consumer. */ + rkq->rkq_rk->rk_type == RD_KAFKA_CONSUMER); +} + /** * @brief Filter out and destroy outdated messages. * @@ -681,7 +760,7 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq, abs_timeout = rd_timeout_init(timeout_ms); - rd_kafka_app_poll_start(rk, 0, timeout_ms); + rd_kafka_app_poll_start(rk, rkq, 0, timeout_ms); rd_kafka_yield_thread = 0; while (cnt < rkmessages_size) { @@ -748,15 +827,15 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq, for (i = cnt - 1; i >= 0; i--) { rko = (rd_kafka_op_t *)rkmessages[i]->_private; - rd_kafka_toppar_t *rktp = rko->rko_rktp; - int64_t offset = rkmessages[i]->offset + 1; - if (unlikely(rktp && (rktp->rktp_app_pos.offset < offset))) - rd_kafka_update_app_pos( - rk, rktp, - RD_KAFKA_FETCH_POS( - offset, - rd_kafka_message_leader_epoch(rkmessages[i])), - RD_DO_LOCK); + rd_kafka_toppar_t *rktp = rko->rko_rktp; + rd_kafka_fetch_pos_t pos = RD_KAFKA_FETCH_POS( + rko->rko_u.fetch.rkm.rkm_rkmessage.offset + 1, + rko->rko_u.fetch.rkm.rkm_u.consumer.leader_epoch); + if (unlikely(rktp && !rko->rko_err && + rko->rko_type == RD_KAFKA_OP_FETCH && + rd_kafka_fetch_pos_cmp(&pos, &rktp->rktp_app_pos) > + 0)) + rd_kafka_update_app_pos(rk, rktp, pos, RD_DO_LOCK); } /* Discard non-desired and already handled ops */ @@ -770,27 +849,133 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq, /* Discard ctrl msgs */ next = TAILQ_FIRST(&ctrl_msg_q); while (next) { - rko = next; - next = TAILQ_NEXT(next, rko_link); - rd_kafka_toppar_t *rktp = rko->rko_rktp; - int64_t offset = rko->rko_u.fetch.rkm.rkm_rkmessage.offset + 1; - if (rktp && (rktp->rktp_app_pos.offset < offset)) - rd_kafka_update_app_pos( - rk, rktp, - RD_KAFKA_FETCH_POS( - offset, - rd_kafka_message_leader_epoch( - &rko->rko_u.fetch.rkm.rkm_rkmessage)), - RD_DO_LOCK); + rko = next; + next = TAILQ_NEXT(next, rko_link); + rd_kafka_toppar_t *rktp = rko->rko_rktp; + rd_kafka_fetch_pos_t pos = RD_KAFKA_FETCH_POS( + rko->rko_u.fetch.rkm.rkm_rkmessage.offset + 1, + rko->rko_u.fetch.rkm.rkm_u.consumer.leader_epoch); + if (rktp && + rd_kafka_fetch_pos_cmp(&pos, &rktp->rktp_app_pos) > 0) + rd_kafka_update_app_pos(rk, rktp, pos, RD_DO_LOCK); rd_kafka_op_destroy(rko); } - rd_kafka_app_polled(rk); + rd_kafka_app_polled(rk, rkq); return cnt; } +/** + * Serve all ops from the share consumer queue. Only CONSUMER_ERR and + * SHARE_FETCH_RESPONSE are enqueued. Processes all available ops: + * - On CONSUMER_ERR: return that error immediately + * - On SHARE_FETCH_RESPONSE: accumulate messages from all responses + */ +rd_kafka_error_t * +rd_kafka_q_serve_share_rkmessages(rd_kafka_q_t *rkq, + int timeout_ms, + rd_kafka_messages_t **rkmessages_out) { + rd_kafka_op_t *rko; + rd_kafka_t *rk = rkq->rkq_rk; + rd_kafka_q_t *fwdq; + rd_ts_t abs_timeout; + rd_kafka_error_t *error = NULL; + rd_kafka_resp_err_t fatal_err; + char fatal_errstr[512]; + + *rkmessages_out = NULL; + + mtx_lock(&rkq->rkq_lock); + if ((fwdq = rd_kafka_q_fwd_get(rkq, 0))) { + mtx_unlock(&rkq->rkq_lock); + error = rd_kafka_q_serve_share_rkmessages(fwdq, timeout_ms, + rkmessages_out); + rd_kafka_q_destroy(fwdq); + return error; + } + mtx_unlock(&rkq->rkq_lock); + + abs_timeout = rd_timeout_init(timeout_ms); + rd_kafka_yield_thread = 0; + + /* Wait for at least one op to arrive */ + mtx_lock(&rkq->rkq_lock); + while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) && + !rd_kafka_q_check_yield(rkq) && + cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock, abs_timeout) == + thrd_success) + ; + rd_kafka_q_mark_served(rkq); + + if (!rko) { + mtx_unlock(&rkq->rkq_lock); + return NULL; + } + + /* Dequeue and process one op */ + rd_kafka_q_deq0(rkq, rko); + mtx_unlock(&rkq->rkq_lock); + + rd_kafka_dbg(rk, QUEUE, "SHAREQ", + "rd_kafka_q_serve_share_rkmessages: " + "dequeued op %s (type %d)", + rd_kafka_op2str(rko->rko_type), rko->rko_type); + + if (rko->rko_type == RD_KAFKA_OP_SHARE_FETCH_RESPONSE) { + /* Allocate the messages handle sized exactly to this op's + * payload — the response may contain more records than + * max.poll.records when multiple partitions are aggregated + * into a single fetch response, so any caller-side + * pre-allocation against max.poll.records would overflow. */ + size_t total_msgs = (size_t)rd_list_cnt( + rko->rko_u.share_fetch_response.message_rkos); + if (total_msgs > 0) { + *rkmessages_out = rd_kafka_messages_new(total_msgs); + rd_kafka_op_process_share_fetch_response( + rko, rkq->rkq_rk->rk_rkshare, + (*rkmessages_out)->elems, 0); + } + rkq->rkq_rk->rk_rkshare->rkshare_fetch_more_records_requested = + rd_false; + } else if (rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) { + /* Return error */ + if (unlikely(rko->rko_err == RD_KAFKA_RESP_ERR__FATAL)) { + /* Never surface the generic __FATAL sentinel naked: + * the specific fatal code + message are set before + * this op is enqueued (and never cleared), so fetch + * them and return a fatal-flagged error. */ + fatal_err = rd_kafka_fatal_error(rk, fatal_errstr, + sizeof(fatal_errstr)); + error = rd_kafka_error_new_fatal(fatal_err, "%s", + fatal_errstr); + } else if (unlikely(rko->rko_error != NULL)) { + error = rko->rko_error; + rko->rko_error = NULL; + } else { + /* Non-fatal share-consumer errors are retriable: the + * app can retry the consume_batch call. */ + error = rd_kafka_error_new_retriable( + rko->rko_err, "%s", + rko->rko_u.err.errstr ? rko->rko_u.err.errstr : ""); + } + } else { + /* Only OP_SHARE_FETCH_RESPONSE and OP_CONSUMER_ERR are + * expected on the share consumer queue. Anything else + * indicates a routing bug or unsupported API misuse; + * abort in devel builds, drop silently in production. */ + rd_kafka_dbg(rk, QUEUE, "SHAREQ", + "Unexpected op %s (type %d) on share " + "consumer queue; dropping", + rd_kafka_op2str(rko->rko_type), rko->rko_type); + rd_dassert(!*"unexpected op on share consumer queue"); + } + + rd_kafka_op_destroy(rko); + return error; +} + void rd_kafka_queue_destroy(rd_kafka_queue_t *rkqu) { if (rkqu->rkqu_is_owner) @@ -897,6 +1082,21 @@ rd_kafka_resp_err_t rd_kafka_set_log_queue(rd_kafka_t *rk, return RD_KAFKA_RESP_ERR_NO_ERROR; } +rd_kafka_error_t *rd_kafka_share_set_log_queue(rd_kafka_share_t *rkshare, + rd_kafka_queue_t *rkqu) { + rd_kafka_resp_err_t err; + + if (!rkshare || !rkshare->rkshare_rk) + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__INVALID_ARG, + "Share consumer handle is NULL or uninitialized"); + + err = rd_kafka_set_log_queue(rkshare->rkshare_rk, rkqu); + + return err ? rd_kafka_error_new(err, "%s", rd_kafka_err2str(err)) + : NULL; +} + void rd_kafka_queue_forward(rd_kafka_queue_t *src, rd_kafka_queue_t *dst) { rd_kafka_q_fwd_set0(src->rkqu_q, dst ? dst->rkqu_q : NULL, 1, /* do_lock */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_queue.h b/lib/librdkafka-2.15.0/src/rdkafka_queue.h similarity index 95% rename from lib/librdkafka-2.10.1/src/rdkafka_queue.h rename to lib/librdkafka-2.15.0/src/rdkafka_queue.h index eb329d1c1d6..9b068ef075a 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_queue.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_queue.h @@ -91,6 +91,12 @@ struct rd_kafka_q_s { * Shall return 1 if op was handled, else 0. */ rd_kafka_q_serve_cb_t *rkq_serve; void *rkq_opaque; + rd_ts_t rkq_ts_last_poll_start; /**< Timestamp of last queue + * poll() call start + * Only relevant for a consumer. */ + rd_ts_t rkq_ts_last_poll_end; /**< Timestamp of last queue + * poll() call end + * Only relevant for a consumer. */ #if ENABLE_DEVEL char rkq_name[64]; /* Debugging: queue name (FUNC:LINE) */ @@ -841,6 +847,13 @@ rd_kafka_op_t *rd_kafka_q_pop_serve(rd_kafka_q_t *rkq, rd_kafka_q_serve_cb_t *callback, void *opaque); rd_kafka_op_t * +rd_kafka_q_pop_serve_maybe_consume(rd_kafka_q_t *rkq, + rd_ts_t timeout_us, + int32_t version, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque); +rd_kafka_op_t * rd_kafka_q_pop(rd_kafka_q_t *rkq, rd_ts_t timeout_us, int32_t version); int rd_kafka_q_serve(rd_kafka_q_t *rkq, int timeout_ms, @@ -848,6 +861,12 @@ int rd_kafka_q_serve(rd_kafka_q_t *rkq, rd_kafka_q_cb_type_t cb_type, rd_kafka_q_serve_cb_t *callback, void *opaque); +int rd_kafka_q_serve_maybe_consume(rd_kafka_q_t *rkq, + int timeout_ms, + int max_cnt, + rd_kafka_q_cb_type_t cb_type, + rd_kafka_q_serve_cb_t *callback, + void *opaque); int rd_kafka_q_move_cnt(rd_kafka_q_t *dstq, @@ -859,6 +878,21 @@ int rd_kafka_q_serve_rkmessages(rd_kafka_q_t *rkq, int timeout_ms, rd_kafka_message_t **rkmessages, size_t rkmessages_size); +/** + * @brief Serve up to one share-consumer op from \p rkq. + * + * On success with a non-empty SHARE_FETCH_RESPONSE op, \c *rkmessages_out + * is set to a newly-allocated rd_kafka_messages_t sized exactly to the op's + * payload. The caller takes ownership and must release it with + * rd_kafka_messages_destroy(). + * + * On error / empty queue / non-fetch ops, \c *rkmessages_out is NULL. + * The returned rd_kafka_error_t (if non-NULL) is owned by the caller. + */ +rd_kafka_error_t * +rd_kafka_q_serve_share_rkmessages(rd_kafka_q_t *rkq, + int timeout_ms, + rd_kafka_messages_t **rkmessages_out); rd_kafka_resp_err_t rd_kafka_q_wait_result(rd_kafka_q_t *rkq, int timeout_ms); int rd_kafka_q_apply(rd_kafka_q_t *rkq, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_range_assignor.c b/lib/librdkafka-2.15.0/src/rdkafka_range_assignor.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_range_assignor.c rename to lib/librdkafka-2.15.0/src/rdkafka_range_assignor.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_request.c b/lib/librdkafka-2.15.0/src/rdkafka_request.c similarity index 95% rename from lib/librdkafka-2.10.1/src/rdkafka_request.c rename to lib/librdkafka-2.15.0/src/rdkafka_request.c index 6fb914c3e96..d6ede7aeaa7 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_request.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_request.c @@ -562,6 +562,9 @@ int rd_kafka_buf_read_NodeEndpoints(rd_kafka_buf_t *rkbuf, int32_t i; rd_kafka_buf_read_arraycnt(rkbuf, &NodeEndpoints->NodeEndpointCnt, RD_KAFKAP_BROKERS_MAX); + // printf(" --------------------------------------- + // rd_kafka_buf_read_NodeEndpoints: NodeEndpointCnt=%d\n", + // NodeEndpoints->NodeEndpointCnt); rd_dassert(!NodeEndpoints->NodeEndpoints); NodeEndpoints->NodeEndpoints = rd_calloc(NodeEndpoints->NodeEndpointCnt, @@ -1163,6 +1166,11 @@ void rd_kafka_OffsetForLeaderEpochRequest( } +int rd_kafka_handle_OffsetFetch_err_action(rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *request) { + return rd_kafka_err_action(rkb, err, request, RD_KAFKA_ERR_ACTION_END); +} /** * Generic handler for OffsetFetch responses. @@ -1362,8 +1370,7 @@ rd_kafka_handle_OffsetFetch(rd_kafka_t *rk, seen_cnt, (*offsets)->cnt, retry_unstable, rd_kafka_err2str(err)); - actions = - rd_kafka_err_action(rkb, err, request, RD_KAFKA_ERR_ACTION_END); + actions = rd_kafka_handle_OffsetFetch_err_action(rkb, err, request); if (actions & RD_KAFKA_ERR_ACTION_REFRESH) { /* Re-query for coordinator */ @@ -1726,9 +1733,9 @@ rd_kafka_handle_OffsetCommit(rd_kafka_t *rk, continue; } + rktpar->err = partition->err; if (partition->err) { - rktpar->err = partition->err; - err = partition->err; + err = partition->err; errcnt++; /* Accumulate actions for per-partition * errors. */ @@ -2438,7 +2445,101 @@ void rd_kafka_ConsumerGroupHeartbeatRequest( rd_kafkap_str_destroy(subscribed_topic_regex_to_send); } +void rd_kafka_ShareGroupHeartbeatRequest( + rd_kafka_broker_t *rkb, + const rd_kafkap_str_t *group_id, + const rd_kafkap_str_t *member_id, + int32_t member_epoch, + const rd_kafkap_str_t *rack_id, + const rd_kafka_topic_partition_list_t *subscribed_topics, + rd_kafka_replyq_t replyq, + rd_kafka_resp_cb_t *resp_cb, + void *opaque) { + rd_kafka_buf_t *rkbuf; + int16_t ApiVersion = 0; + int features; + size_t rkbuf_size = 0; + + ApiVersion = rd_kafka_broker_ApiVersion_supported( + rkb, RD_KAFKAP_ShareGroupHeartbeat, 1, 1, &features); + + rd_rkb_dbg(rkb, CGRP, "SHAREHEARTBEAT", + "ShareGroupHeartbeat version %d for group \"%s\", member id " + "\"%s\", topic count = %d", + ApiVersion, group_id ? group_id->str : "NULL", + member_id ? member_id->str : "NULL", + subscribed_topics ? subscribed_topics->cnt : -1); + if (ApiVersion == -1) { + rd_kafka_cgrp_coord_dead(rkb->rkb_rk->rk_cgrp, + RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE, + "ShareGroupHeartbeatRequest not " + "supported by broker"); + return; + } + + // debug log all the fields + if (rd_rkb_is_dbg(rkb, CGRP)) { + char subscribed_topics_str[512] = "NULL"; + if (subscribed_topics) { + rd_kafka_topic_partition_list_str( + subscribed_topics, subscribed_topics_str, + sizeof(subscribed_topics_str), 0); + } + rd_rkb_dbg(rkb, CGRP, "SHAREHEARTBEAT", + "ShareGroupHeartbeat of group id \"%s\", " + "member id \"%s\", member epoch %d, rack id \"%s\"" + ", subscribed topics \"%s\"", + group_id ? group_id->str : "NULL", + member_id ? member_id->str : "NULL", member_epoch, + rack_id ? rack_id->str : "NULL", + subscribed_topics_str); + } + + if (group_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(group_id); + if (member_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(member_id); + rkbuf_size += 4; /* MemberEpoch */ + if (rack_id) + rkbuf_size += RD_KAFKAP_STR_SIZE(rack_id); + if (subscribed_topics) { + rkbuf_size += + ((subscribed_topics->cnt * (4 + 50)) + 4 /* array size */); + } + + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_ShareGroupHeartbeat, 1, rkbuf_size, rd_true); + + rd_kafka_buf_write_kstr(rkbuf, group_id); + rd_kafka_buf_write_kstr(rkbuf, member_id); + rd_kafka_buf_write_i32(rkbuf, member_epoch); + rd_kafka_buf_write_kstr(rkbuf, rack_id); + + if (subscribed_topics) { + int topics_cnt = subscribed_topics->cnt; + + /* write Topics */ + rd_kafka_buf_write_arraycnt(rkbuf, topics_cnt); + while (--topics_cnt >= 0) { + if (rd_rkb_is_dbg(rkb, CGRP)) + rd_rkb_dbg( + rkb, CGRP, "SHAREHEARTBEAT", + "ShareGroupHeartbeat subscribed " + "topic %s", + subscribed_topics->elems[topics_cnt].topic); + rd_kafka_buf_write_str( + rkbuf, subscribed_topics->elems[topics_cnt].topic, + -1); + } + } else { + rd_kafka_buf_write_arraycnt(rkbuf, -1); + } + + rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, features); + + rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque); +} /** * @brief Construct and send ListGroupsRequest to \p rkb @@ -2535,7 +2636,7 @@ rd_kafka_DescribeGroupsRequest(rd_kafka_broker_t *rkb, size_t of_GroupsArrayCnt; if (max_ApiVersion < 0) - max_ApiVersion = 4; + max_ApiVersion = 5; if (max_ApiVersion > ApiVersion) { /* Remark: don't check if max_ApiVersion is zero. @@ -2554,9 +2655,9 @@ rd_kafka_DescribeGroupsRequest(rd_kafka_broker_t *rkb, rkbuf = rd_kafka_buf_new_flexver_request( rkb, RD_KAFKAP_DescribeGroups, 1, 4 /* rd_kafka_buf_write_arraycnt_pos */ + - 1 /* IncludeAuthorizedOperations */ + 1 /* tags */ + + 1 /* IncludeAuthorizedOperations */ + 32 * group_cnt /* Groups */, - rd_false); + ApiVersion >= 5 /* is_flexver */); /* write Groups */ of_GroupsArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf); @@ -2620,6 +2721,7 @@ static void rd_kafka_handle_Metadata(rd_kafka_t *rk, if (err) goto err; + rd_kafka_rebootstrap_tmr_stop(rk); if (rko && rko->rko_replyq.q) { /* Reply to metadata requester, passing on the metadata. * Reuse requesting rko for the reply. */ @@ -2638,12 +2740,18 @@ static void rd_kafka_handle_Metadata(rd_kafka_t *rk, goto done; err: - actions = rd_kafka_err_action(rkb, err, request, + actions = rd_kafka_err_action( + rkb, err, request, - RD_KAFKA_ERR_ACTION_RETRY, - RD_KAFKA_RESP_ERR__PARTIAL, + RD_KAFKA_ERR_ACTION_SPECIAL, RD_KAFKA_RESP_ERR_REBOOTSTRAP_REQUIRED, - RD_KAFKA_ERR_ACTION_END); + RD_KAFKA_ERR_ACTION_RETRY, RD_KAFKA_RESP_ERR__PARTIAL, + + RD_KAFKA_ERR_ACTION_END); + + if (actions & RD_KAFKA_ERR_ACTION_SPECIAL) { + rd_kafka_rebootstrap(rk); + } if (actions & RD_KAFKA_ERR_ACTION_RETRY) { /* In case it's a brokers full refresh call, @@ -2738,6 +2846,18 @@ static void rd_kafka_handle_Metadata(rd_kafka_t *rk, * * @sa rd_kafka_MetadataRequest(). * @sa rd_kafka_MetadataRequest_resp_cb(). + * + * @locality any + * @locks none + * @locks_acquired + * - mtx_lock(&rkb->rkb_rk->rk_metadata_cache.rkmc_full_lock) in + * this function + * - rd_kafka_broker_lock() in + * rd_kafka_broker_ApiVersion_supported() + * - rd_kafka_timers_lock() in + * rd_kafka_rebootstrap_tmr_start_maybe() + * - mtx_lock(&rkq->rkq_lock) in + * rd_kafka_broker_buf_enq_replyq() */ static rd_kafka_resp_err_t rd_kafka_MetadataRequest0(rd_kafka_broker_t *rkb, @@ -2765,7 +2885,7 @@ rd_kafka_MetadataRequest0(rd_kafka_broker_t *rkb, int *full_incr = NULL; void *handler_arg = NULL; rd_kafka_resp_cb_t *handler_cb = rd_kafka_handle_Metadata; - int16_t metadata_max_version = 12; + int16_t metadata_max_version = 13; rd_kafka_replyq_t use_replyq = replyq; /* In case we want cluster authorized operations in the Metadata @@ -2871,9 +2991,28 @@ rd_kafka_MetadataRequest0(rd_kafka_broker_t *rkb, rkbuf->rkbuf_u.Metadata.decr = full_incr; rkbuf->rkbuf_u.Metadata.decr_lock = &rkb->rkb_rk->rk_metadata_cache.rkmc_full_lock; + } else if (!resp_cb) { + /* + * Conditions for starting the rebootstrap timer: + * + * !full_incr: + * In case it's a full request, forced or not, it won't be + * retried on the same broker to avoid blocking metadata + * requests, because of the lock, when that broker isn't + * available. We don't start the timer as we cannot ensure the + * request is retried for the duration of + * `metadata.recovery.rebootstrap.trigger.ms`. + * + * !resp_cb: + * Same reasoning applies when we use a custom callback, for the + * AdminClient requests for example. + * + * Start the rebootstrap timer only if it's the first + * metadata refresh request after last successful response, + * so the timer is not reset if already scheduled. */ + rd_kafka_rebootstrap_tmr_start_maybe(rkb->rkb_rk); } - if (topic_cnt > 0) { char *topic; int i; @@ -3590,6 +3729,107 @@ void rd_kafkap_leader_discovery_set_CurrentLeader( mdpi->id = partition_id; mdpi->leader_epoch = CurrentLeader->LeaderEpoch; } + +/** + * @brief Apply inline metadata updates from a Share* response. + * + * Builds a synthetic metadata-internal struct from collected + * per-partition CurrentLeader hints plus top-level NodeEndpoints, + * and enqueues it as OP_METADATA_UPDATE. + * + * Entries in \p leader_changes must be grouped by topic_id in + * iteration order — naturally satisfied when a parser walks + * topic-then-partition. + * + * @locality broker thread + */ +void rd_kafkap_share_leader_changes_apply( + rd_kafka_broker_t *rkb, + rd_list_t *leader_changes, + rd_kafkap_NodeEndpoints_t *NodeEndpoints) { + rd_kafkap_share_leader_change_t *lc; + int *partitions_per_topic; + int distinct_topic_cnt; + rd_kafka_Uuid_t prev_topic; + rd_tmpabuf_t tbuf; + rd_kafka_metadata_internal_t *mdi; + rd_kafka_op_t *rko; + int32_t nodeid; + int k, t_idx, p_idx; + + if (!leader_changes || rd_list_cnt(leader_changes) == 0 || + NodeEndpoints->NodeEndpointCnt == 0) + return; + + rd_rkb_dbg(rkb, METADATA, "LEADER", + "Applying inline metadata update from Share response: " + "%d partition leader change(s), %" PRId32 + " node endpoint(s)", + rd_list_cnt(leader_changes), NodeEndpoints->NodeEndpointCnt); + + partitions_per_topic = + rd_calloc(rd_list_cnt(leader_changes), sizeof(int)); + + prev_topic = RD_KAFKA_UUID_ZERO; + t_idx = -1; + RD_LIST_FOREACH(lc, leader_changes, k) { + if (t_idx < 0 || + rd_kafka_Uuid_cmp(lc->topic_id, prev_topic) != 0) { + t_idx++; + prev_topic = lc->topic_id; + } + partitions_per_topic[t_idx]++; + } + distinct_topic_cnt = t_idx + 1; + + rd_kafka_broker_lock(rkb); + nodeid = rkb->rkb_nodeid; + rd_kafka_broker_unlock(rkb); + + rd_tmpabuf_new(&tbuf, 0, rd_true /*assert on fail*/); + rd_tmpabuf_add_alloc(&tbuf, sizeof(*mdi)); + rd_kafkap_leader_discovery_tmpabuf_add_alloc_brokers(&tbuf, + NodeEndpoints); + rd_kafkap_leader_discovery_tmpabuf_add_alloc_topics(&tbuf, + distinct_topic_cnt); + for (t_idx = 0; t_idx < distinct_topic_cnt; t_idx++) + rd_kafkap_leader_discovery_tmpabuf_add_alloc_topic( + &tbuf, NULL, partitions_per_topic[t_idx]); + rd_tmpabuf_finalize(&tbuf); + + mdi = rd_tmpabuf_alloc(&tbuf, sizeof(*mdi)); + rd_kafkap_leader_discovery_metadata_init(mdi, nodeid); + rd_kafkap_leader_discovery_set_brokers(&tbuf, mdi, NodeEndpoints); + rd_kafkap_leader_discovery_set_topic_cnt(&tbuf, mdi, + distinct_topic_cnt); + + prev_topic = RD_KAFKA_UUID_ZERO; + t_idx = -1; + p_idx = 0; + RD_LIST_FOREACH(lc, leader_changes, k) { + if (t_idx < 0 || + rd_kafka_Uuid_cmp(lc->topic_id, prev_topic) != 0) { + t_idx++; + p_idx = 0; + prev_topic = lc->topic_id; + rd_kafkap_leader_discovery_set_topic( + &tbuf, mdi, t_idx, lc->topic_id, NULL, + partitions_per_topic[t_idx]); + } + rd_kafkap_leader_discovery_set_CurrentLeader( + &tbuf, mdi, t_idx, p_idx, lc->partition, + &lc->current_leader); + p_idx++; + } + + rd_free(partitions_per_topic); + + rko = rd_kafka_op_new(RD_KAFKA_OP_METADATA_UPDATE); + rko->rko_u.metadata.md = &mdi->metadata; + rko->rko_u.metadata.mdi = mdi; + rd_kafka_q_enq(rkb->rkb_rk->rk_ops, rko); +} + /**@}*/ static int rd_kafkap_Produce_reply_tags_partition_parse( @@ -5080,7 +5320,7 @@ rd_kafka_DeleteTopicsRequest(rd_kafka_broker_t *rkb, } ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_DeleteTopics, 0, 1, &features); + rkb, RD_KAFKAP_DeleteTopics, 0, 4, &features); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "Topic Admin API (KIP-4) not supported " @@ -5089,13 +5329,13 @@ rd_kafka_DeleteTopicsRequest(rd_kafka_broker_t *rkb, return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE; } - rkbuf = - rd_kafka_buf_new_request(rkb, RD_KAFKAP_DeleteTopics, 1, - /* FIXME */ - 4 + (rd_list_cnt(del_topics) * 100) + 4); + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_DeleteTopics, 1, + /* FIXME */ + 4 + (rd_list_cnt(del_topics) * 100) + 4, ApiVersion >= 4); /* #topics */ - rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(del_topics)); + rd_kafka_buf_write_arraycnt(rkbuf, rd_list_cnt(del_topics)); while ((delt = rd_list_elem(del_topics, i++))) rd_kafka_buf_write_str(rkbuf, delt->topic, -1); @@ -5149,7 +5389,7 @@ rd_kafka_DeleteRecordsRequest(rd_kafka_broker_t *rkb, partitions = rd_list_elem(offsets_list, 0); ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_DeleteRecords, 0, 1, &features); + rkb, RD_KAFKAP_DeleteRecords, 0, 2, &features); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "DeleteRecords Admin API (KIP-107) not supported " @@ -5157,8 +5397,9 @@ rd_kafka_DeleteRecordsRequest(rd_kafka_broker_t *rkb, return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE; } - rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_DeleteRecords, 1, - 4 + (partitions->cnt * 100) + 4); + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_DeleteRecords, 1, 4 + (partitions->cnt * 100) + 4, + ApiVersion >= 2); const rd_kafka_topic_partition_field_t fields[] = { RD_KAFKA_TOPIC_PARTITION_FIELD_PARTITION, @@ -5219,7 +5460,7 @@ rd_kafka_CreatePartitionsRequest(rd_kafka_broker_t *rkb, } ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_CreatePartitions, 0, 0, NULL); + rkb, RD_KAFKAP_CreatePartitions, 0, 2, NULL); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "CreatePartitions (KIP-195) not supported " @@ -5228,12 +5469,12 @@ rd_kafka_CreatePartitionsRequest(rd_kafka_broker_t *rkb, return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE; } - rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_CreatePartitions, 1, - 4 + (rd_list_cnt(new_parts) * 200) + - 4 + 1); + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_CreatePartitions, 1, + 4 + (rd_list_cnt(new_parts) * 200) + 4 + 1, ApiVersion >= 2); /* #topics */ - rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(new_parts)); + rd_kafka_buf_write_arraycnt(rkbuf, rd_list_cnt(new_parts)); while ((newp = rd_list_elem(new_parts, i++))) { /* topic */ @@ -5244,21 +5485,21 @@ rd_kafka_CreatePartitionsRequest(rd_kafka_broker_t *rkb, /* #replica_assignment */ if (rd_list_empty(&newp->replicas)) { - rd_kafka_buf_write_i32(rkbuf, -1); + rd_kafka_buf_write_arraycnt(rkbuf, -1); } else { const rd_list_t *replicas; int pi = -1; - rd_kafka_buf_write_i32(rkbuf, - rd_list_cnt(&newp->replicas)); + rd_kafka_buf_write_arraycnt( + rkbuf, rd_list_cnt(&newp->replicas)); while ( (replicas = rd_list_elem(&newp->replicas, ++pi))) { int ri = 0; /* replica count */ - rd_kafka_buf_write_i32(rkbuf, - rd_list_cnt(replicas)); + rd_kafka_buf_write_arraycnt( + rkbuf, rd_list_cnt(replicas)); /* replica */ for (ri = 0; ri < rd_list_cnt(replicas); ri++) { @@ -5266,8 +5507,12 @@ rd_kafka_CreatePartitionsRequest(rd_kafka_broker_t *rkb, rkbuf, rd_list_get_int32(replicas, ri)); } + + rd_kafka_buf_write_tags_empty(rkbuf); } } + + rd_kafka_buf_write_tags_empty(rkbuf); } /* timeout */ @@ -5595,7 +5840,7 @@ rd_kafka_DeleteGroupsRequest(rd_kafka_broker_t *rkb, rd_kafka_DeleteGroup_t *delt; ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_DeleteGroups, 0, 1, &features); + rkb, RD_KAFKAP_DeleteGroups, 0, 2, &features); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "DeleteGroups Admin API (KIP-229) not supported " @@ -5604,12 +5849,12 @@ rd_kafka_DeleteGroupsRequest(rd_kafka_broker_t *rkb, return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE; } - rkbuf = - rd_kafka_buf_new_request(rkb, RD_KAFKAP_DeleteGroups, 1, - 4 + (rd_list_cnt(del_groups) * 100) + 4); + rkbuf = rd_kafka_buf_new_flexver_request( + rkb, RD_KAFKAP_DeleteGroups, 1, + 4 + (rd_list_cnt(del_groups) * 100) + 4, ApiVersion >= 2); /* #groups */ - rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(del_groups)); + rd_kafka_buf_write_arraycnt(rkbuf, rd_list_cnt(del_groups)); while ((delt = rd_list_elem(del_groups, i++))) rd_kafka_buf_write_str(rkbuf, delt->group, -1); @@ -5672,7 +5917,7 @@ rd_kafka_CreateAclsRequest(rd_kafka_broker_t *rkb, } ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_CreateAcls, 0, 1, NULL); + rkb, RD_KAFKAP_CreateAcls, 0, 2, NULL); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "ACLs Admin API (KIP-140) not supported " @@ -5713,10 +5958,11 @@ rd_kafka_CreateAclsRequest(rd_kafka_broker_t *rkb, len += rd_kafka_AclBinding_request_size(new_acl, ApiVersion); } - rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_CreateAcls, 1, len); + rkbuf = rd_kafka_buf_new_flexver_request(rkb, RD_KAFKAP_CreateAcls, 1, + len, ApiVersion >= 2); /* #acls */ - rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(new_acls)); + rd_kafka_buf_write_arraycnt(rkbuf, rd_list_cnt(new_acls)); RD_LIST_FOREACH(new_acl, new_acls, i) { rd_kafka_buf_write_i8(rkbuf, new_acl->restype); @@ -5735,6 +5981,8 @@ rd_kafka_CreateAclsRequest(rd_kafka_broker_t *rkb, rd_kafka_buf_write_i8(rkbuf, new_acl->operation); rd_kafka_buf_write_i8(rkbuf, new_acl->permission_type); + + rd_kafka_buf_write_tags_empty(rkbuf); } /* timeout */ @@ -5791,7 +6039,7 @@ rd_kafka_resp_err_t rd_kafka_DescribeAclsRequest( acl = rd_list_elem(acls, 0); ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_DescribeAcls, 0, 1, NULL); + rkb, RD_KAFKAP_DescribeAcls, 0, 2, NULL); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "ACLs Admin API (KIP-140) not supported " @@ -5821,9 +6069,9 @@ rd_kafka_resp_err_t rd_kafka_DescribeAclsRequest( } } - rkbuf = rd_kafka_buf_new_request( + rkbuf = rd_kafka_buf_new_flexver_request( rkb, RD_KAFKAP_DescribeAcls, 1, - rd_kafka_AclBinding_request_size(acl, ApiVersion)); + rd_kafka_AclBinding_request_size(acl, ApiVersion), ApiVersion >= 2); /* resource_type */ rd_kafka_buf_write_i8(rkbuf, acl->restype); @@ -5896,7 +6144,7 @@ rd_kafka_DeleteAclsRequest(rd_kafka_broker_t *rkb, } ApiVersion = rd_kafka_broker_ApiVersion_supported( - rkb, RD_KAFKAP_DeleteAcls, 0, 1, NULL); + rkb, RD_KAFKAP_DeleteAcls, 0, 2, NULL); if (ApiVersion == -1) { rd_snprintf(errstr, errstr_size, "ACLs Admin API (KIP-140) not supported " @@ -5932,10 +6180,11 @@ rd_kafka_DeleteAclsRequest(rd_kafka_broker_t *rkb, len += rd_kafka_AclBinding_request_size(acl, ApiVersion); } - rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_DeleteAcls, 1, len); + rkbuf = rd_kafka_buf_new_flexver_request(rkb, RD_KAFKAP_DeleteAcls, 1, + len, ApiVersion >= 2); /* #acls */ - rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(del_acls)); + rd_kafka_buf_write_arraycnt(rkbuf, rd_list_cnt(del_acls)); RD_LIST_FOREACH(acl, del_acls, i) { /* resource_type */ @@ -5962,6 +6211,8 @@ rd_kafka_DeleteAclsRequest(rd_kafka_broker_t *rkb, /* permission type (rd_kafka_AclPermissionType_t) */ rd_kafka_buf_write_i8(rkbuf, acl->permission_type); + + rd_kafka_buf_write_tags_empty(rkbuf); } /* timeout */ @@ -6511,15 +6762,14 @@ rd_kafka_PushTelemetryRequest(rd_kafka_broker_t *rkb, } size_t len = sizeof(rd_kafka_Uuid_t) + sizeof(int32_t) + - sizeof(rd_bool_t) + sizeof(compression_type) + - metrics_size; + sizeof(rd_bool_t) + sizeof(int8_t) + metrics_size; rkbuf = rd_kafka_buf_new_flexver_request(rkb, RD_KAFKAP_PushTelemetry, 1, len, rd_true); rd_kafka_buf_write_uuid(rkbuf, client_instance_id); rd_kafka_buf_write_i32(rkbuf, subscription_id); rd_kafka_buf_write_bool(rkbuf, terminating); - rd_kafka_buf_write_i8(rkbuf, compression_type); + rd_kafka_buf_write_i8(rkbuf, (int8_t)compression_type); rd_dassert(metrics != NULL); rd_dassert(metrics_size >= 0); @@ -6584,10 +6834,12 @@ void rd_kafka_handle_GetTelemetrySubscriptions(rd_kafka_t *rk, rk->rk_telemetry.accepted_compression_types = rd_calloc(arraycnt, sizeof(rd_kafka_compression_t)); - for (i = 0; i < (size_t)arraycnt; i++) - rd_kafka_buf_read_i8( - rkbuf, - &rk->rk_telemetry.accepted_compression_types[i]); + for (i = 0; i < (size_t)arraycnt; i++) { + int8_t AcceptedCompressionType; + rd_kafka_buf_read_i8(rkbuf, &AcceptedCompressionType); + rk->rk_telemetry.accepted_compression_types[i] = + AcceptedCompressionType; + } } else { rk->rk_telemetry.accepted_compression_types_cnt = 1; rk->rk_telemetry.accepted_compression_types = diff --git a/lib/librdkafka-2.10.1/src/rdkafka_request.h b/lib/librdkafka-2.15.0/src/rdkafka_request.h similarity index 96% rename from lib/librdkafka-2.10.1/src/rdkafka_request.h rename to lib/librdkafka-2.15.0/src/rdkafka_request.h index 68932b309ee..d3090d03fb5 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_request.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_request.h @@ -245,6 +245,9 @@ void rd_kafka_OffsetForLeaderEpochRequest( rd_kafka_resp_cb_t *resp_cb, void *opaque); +int rd_kafka_handle_OffsetFetch_err_action(rd_kafka_broker_t *rkb, + rd_kafka_resp_err_t err, + rd_kafka_buf_t *request); rd_kafka_resp_err_t rd_kafka_handle_OffsetFetch(rd_kafka_t *rk, @@ -397,6 +400,17 @@ void rd_kafka_ConsumerGroupHeartbeatRequest( rd_kafka_resp_cb_t *resp_cb, void *opaque); +void rd_kafka_ShareGroupHeartbeatRequest( + rd_kafka_broker_t *rkb, + const rd_kafkap_str_t *group_id, + const rd_kafkap_str_t *member_id, + int32_t member_epoch, + const rd_kafkap_str_t *rack_id, + const rd_kafka_topic_partition_list_t *subscribed_topics, + rd_kafka_replyq_t replyq, + rd_kafka_resp_cb_t *resp_cb, + void *opaque); + rd_kafka_resp_err_t rd_kafka_MetadataRequest(rd_kafka_broker_t *rkb, const rd_list_t *topics, rd_list_t *topic_ids, @@ -689,6 +703,24 @@ void rd_kafkap_leader_discovery_set_CurrentLeader( int32_t partition_id, rd_kafkap_CurrentLeader_t *CurrentLeader); +/** + * @brief Per-partition leader-change record collected from a Share* + * response for inline metadata update. + * + * Populated by Share* response parsers and consumed by + * rd_kafkap_share_leader_changes_apply(). + */ +typedef struct rd_kafkap_share_leader_change_s { + rd_kafka_Uuid_t topic_id; + int32_t partition; + rd_kafkap_CurrentLeader_t current_leader; +} rd_kafkap_share_leader_change_t; + +void rd_kafkap_share_leader_changes_apply( + rd_kafka_broker_t *rkb, + rd_list_t *leader_changes, + rd_kafkap_NodeEndpoints_t *NodeEndpoints); + rd_kafka_resp_err_t rd_kafka_GetTelemetrySubscriptionsRequest(rd_kafka_broker_t *rkb, char *errstr, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_roundrobin_assignor.c b/lib/librdkafka-2.15.0/src/rdkafka_roundrobin_assignor.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_roundrobin_assignor.c rename to lib/librdkafka-2.15.0/src/rdkafka_roundrobin_assignor.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl.c similarity index 85% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl.c rename to lib/librdkafka-2.15.0/src/rdkafka_sasl.c index 32ebe3b198e..aab16f6d506 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_sasl.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_sasl.c @@ -172,12 +172,14 @@ int rd_kafka_sasl_io_event(rd_kafka_transport_t *rktrans, r = rd_kafka_transport_framed_recv(rktrans, &rkbuf, errstr, errstr_size); if (r == -1) { - if (!strcmp(errstr, "Disconnected")) - rd_snprintf(errstr, errstr_size, - "Disconnected: check client %s credentials " + if (rd_kafka_transport_error_disconnected(errstr)) { + int curr_len = strlen(errstr); + rd_snprintf(errstr + curr_len, errstr_size - curr_len, + ": check client %s credentials " "and broker logs", rktrans->rktrans_rkb->rkb_rk->rk_conf.sasl .mechanisms); + } return -1; } else if (r == 0) /* not fully received yet */ return 0; @@ -474,6 +476,65 @@ rd_kafka_error_t *rd_kafka_sasl_background_callbacks_enable(rd_kafka_t *rk) { } +rd_kafka_error_t * +rd_kafka_share_sasl_background_callbacks_enable(rd_kafka_share_t *rkshare) { + return rd_kafka_sasl_background_callbacks_enable(rkshare->rkshare_rk); +} + + +rd_kafka_error_t *rd_kafka_share_sasl_set_credentials(rd_kafka_share_t *rkshare, + const char *username, + const char *password) { + return rd_kafka_sasl_set_credentials(rkshare->rkshare_rk, username, + password); +} + + +/* TODO KIP-932: add integration tests for the share-consumer OAUTHBEARER + * SASL APIs below (set_token, set_token_failure, queue_get_sasl). */ + +rd_kafka_queue_t *rd_kafka_share_queue_get_sasl(rd_kafka_share_t *rkshare) { + if (unlikely(!rkshare || !rkshare->rkshare_rk)) + return NULL; + return rd_kafka_queue_get_sasl(rkshare->rkshare_rk); +} + + +rd_kafka_resp_err_t +rd_kafka_share_oauthbearer_set_token(rd_kafka_share_t *rkshare, + const char *token_value, + int64_t md_lifetime_ms, + const char *md_principal_name, + const char **extensions, + size_t extension_size, + char *errstr, + size_t errstr_size) { + if (unlikely(!rkshare || !rkshare->rkshare_rk)) { + rd_snprintf(errstr, errstr_size, + "Share consumer handle is NULL or uninitialized"); + return RD_KAFKA_RESP_ERR__INVALID_ARG; + } + if (unlikely(!token_value || !md_principal_name)) { + rd_snprintf(errstr, errstr_size, "%s must not be NULL", + !token_value ? "token_value" : "md_principal_name"); + return RD_KAFKA_RESP_ERR__INVALID_ARG; + } + return rd_kafka_oauthbearer_set_token( + rkshare->rkshare_rk, token_value, md_lifetime_ms, md_principal_name, + extensions, extension_size, errstr, errstr_size); +} + + +rd_kafka_resp_err_t +rd_kafka_share_oauthbearer_set_token_failure(rd_kafka_share_t *rkshare, + const char *errstr) { + if (unlikely(!rkshare || !rkshare->rkshare_rk)) + return RD_KAFKA_RESP_ERR__INVALID_ARG; + return rd_kafka_oauthbearer_set_token_failure(rkshare->rkshare_rk, + errstr); +} + + /** * Global SASL termination. */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl.h b/lib/librdkafka-2.15.0/src/rdkafka_sasl.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl.h rename to lib/librdkafka-2.15.0/src/rdkafka_sasl.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_cyrus.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl_cyrus.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_cyrus.c rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_cyrus.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_int.h b/lib/librdkafka-2.15.0/src/rdkafka_sasl_int.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_int.h rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_int.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer.c similarity index 99% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer.c rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer.c index e0b7e532ce9..daf77da8fb3 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer.c @@ -1256,8 +1256,13 @@ static int rd_kafka_sasl_oauthbearer_client_new(rd_kafka_transport_t *rktrans, return -1; } - state->token_value = rd_strdup(handle->token_value); - state->md_principal_name = rd_strdup(handle->md_principal_name); + state->token_value = rd_strdup(handle->token_value); + if (handle->md_principal_name) + state->md_principal_name = rd_strdup(handle->md_principal_name); + else + state->md_principal_name = NULL; + + rd_list_init_copy(&state->extensions, &handle->extensions); rd_list_copy_to(&state->extensions, &handle->extensions, rd_strtup_list_copy, NULL); @@ -1333,13 +1338,11 @@ static int rd_kafka_sasl_oauthbearer_init(rd_kafka_t *rk, #if WITH_OAUTHBEARER_OIDC if (rk->rk_conf.sasl.oauthbearer.method == RD_KAFKA_SASL_OAUTHBEARER_METHOD_OIDC && - rk->rk_conf.sasl.oauthbearer.token_refresh_cb == - rd_kafka_oidc_token_refresh_cb) { + rk->rk_conf.sasl.oauthbearer.builtin_token_refresh_cb) { handle->internal_refresh = rd_true; rd_kafka_sasl_background_callbacks_enable(rk); } #endif - /* Otherwise enqueue a refresh callback for the application. */ rd_kafka_oauthbearer_enqueue_token_refresh(handle); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer.h b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer.h rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer.h diff --git a/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer_oidc.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer_oidc.c new file mode 100644 index 00000000000..6dca2df347c --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer_oidc.c @@ -0,0 +1,1827 @@ +/* + * librdkafka - The Apache Kafka C/C++ library + * + * Copyright (c) 2021-2022, Magnus Edenhill + * 2023, Confluent Inc. + + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +/** + * Builtin SASL OAUTHBEARER OIDC support + */ +#include "rdkafka_int.h" +#include "rdkafka_sasl_int.h" +#include "rdunittest.h" +#include "cJSON.h" +#include +#include "rdhttp.h" +#include "rdkafka_sasl_oauthbearer_oidc.h" +#include "rdbase64.h" + + +/** + * @brief Generate Authorization field for HTTP header. + * The field contains base64-encoded string which + * is generated from \p client_id and \p client_secret. + * + * @returns Return the authorization field. + * + * @locality Any thread. + */ +static char * +rd_kafka_oidc_client_credentials_build_auth_header(const char *client_id, + const char *client_secret) { + + rd_chariov_t client_authorization_in; + rd_chariov_t client_authorization_out; + + size_t authorization_base64_header_size; + char *authorization_base64_header; + + client_authorization_in.size = + strlen(client_id) + strlen(client_secret) + 2; + client_authorization_in.ptr = rd_malloc(client_authorization_in.size); + rd_snprintf(client_authorization_in.ptr, client_authorization_in.size, + "%s:%s", client_id, client_secret); + + client_authorization_in.size--; + rd_base64_encode(&client_authorization_in, &client_authorization_out); + rd_assert(client_authorization_out.ptr); + + authorization_base64_header_size = + strlen("Authorization: Basic ") + client_authorization_out.size + 1; + authorization_base64_header = + rd_malloc(authorization_base64_header_size); + rd_snprintf(authorization_base64_header, + authorization_base64_header_size, "Authorization: Basic %s", + client_authorization_out.ptr); + + rd_free(client_authorization_in.ptr); + rd_free(client_authorization_out.ptr); + return authorization_base64_header; +} + + +/** + * @brief Build headers for HTTP(S) requests based on \p client_id + * and \p client_secret. The result will be returned in \p *headersp. + * + * @locality Any thread. + */ +static void +rd_kafka_oidc_client_credentials_build_headers(const char *client_id, + const char *client_secret, + struct curl_slist **headersp) { + char *authorization_base64_header; + + authorization_base64_header = + rd_kafka_oidc_client_credentials_build_auth_header(client_id, + client_secret); + + *headersp = curl_slist_append(*headersp, "Accept: application/json"); + *headersp = curl_slist_append(*headersp, authorization_base64_header); + + *headersp = curl_slist_append( + *headersp, "Content-Type: application/x-www-form-urlencoded"); + + rd_free(authorization_base64_header); +} + +/** + * @brief The format of JWT is Header.Payload.Signature. + * Extract and decode payloads from JWT \p src. + * The decoded payloads will be returned in \p *bufplainp. + * + * @returns Return error message while decoding the payload. + */ +static const char *rd_kafka_jwt_b64_decode_payload(const char *src, + char **bufplainp) { + char *converted_src; + char *payload = NULL; + + const char *errstr = NULL; + + int i, padding, len; + + int payload_len; + int nbytesdecoded; + + int payloads_start = 0; + int payloads_end = 0; + + len = (int)strlen(src); + converted_src = rd_malloc(len + 4); + + for (i = 0; i < len; i++) { + switch (src[i]) { + case '-': + converted_src[i] = '+'; + break; + + case '_': + converted_src[i] = '/'; + break; + + case '.': + if (payloads_start == 0) + payloads_start = i + 1; + else { + if (payloads_end > 0) { + errstr = + "The token is invalid with more " + "than 2 delimiters"; + goto done; + } + payloads_end = i; + } + /* FALLTHRU */ + + default: + converted_src[i] = src[i]; + } + } + + if (payloads_start == 0 || payloads_end == 0) { + errstr = "The token is invalid with less than 2 delimiters"; + goto done; + } + + payload_len = payloads_end - payloads_start; + payload = rd_malloc(payload_len + 4); + strncpy(payload, (converted_src + payloads_start), payload_len); + + padding = 4 - (payload_len % 4); + if (padding < 4) { + while (padding--) + payload[payload_len++] = '='; + } + + nbytesdecoded = ((payload_len + 3) / 4) * 3; + *bufplainp = rd_malloc(nbytesdecoded + 1); + + if (EVP_DecodeBlock((uint8_t *)(*bufplainp), (uint8_t *)payload, + (int)payload_len) == -1) { + errstr = "Failed to decode base64 payload"; + } + +done: + RD_IF_FREE(payload, rd_free); + RD_IF_FREE(converted_src, rd_free); + return errstr; +} + +/** + * @brief Build post_fields with \p scope. + * The format of the post_fields is + * `grant_type=client_credentials&scope=scope` + * The post_fields will be returned in \p *post_fields. + * The post_fields_size will be returned in \p post_fields_size. + * + */ +static void +rd_kafka_oidc_client_credentials_build_post_fields(const char *scope, + char **post_fields, + size_t *post_fields_size) { + size_t scope_size = 0; + + if (scope) + scope_size = strlen(scope); + if (scope_size == 0) { + *post_fields = rd_strdup("grant_type=client_credentials"); + *post_fields_size = strlen("grant_type=client_credentials"); + } else { + *post_fields_size = + strlen("grant_type=client_credentials&scope=") + scope_size; + *post_fields = rd_malloc(*post_fields_size + 1); + rd_snprintf(*post_fields, *post_fields_size + 1, + "grant_type=client_credentials&scope=%s", scope); + } +} + +/** + * @brief Get JWT algorithm label string for the specified signing algorithm. + * + * @param token_signing_algo The algorithm enum value + * + * @returns String representation of the algorithm. + * + * @locality Any thread. + */ +static char *rd_kafka_oidc_assertion_get_algo_label( + const rd_kafka_oauthbearer_assertion_algorithm_t token_signing_algo) { + switch (token_signing_algo) { + case RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_RS256: + return "RS256"; + case RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_ES256: + return "ES256"; + default: + rd_assert(!*"Unknown JOSE algorithm"); + return NULL; + } +} + +/** + * @brief Parse a JWT template file and extract header and payload JSON + * objects. + * + * Reads and parses the JWT template file, which should contain a JSON object + * with "header" and "payload" properties. + * + * @param rk + * @param jwt_template_file_path Path to the template file + * @param header Pointer to store the parsed header JSON object + * @param payload Pointer to store the parsed payload JSON object + * + * @returns 0 on success, -1 on failure + * + * @locality Any thread. + */ +static int +rd_kafka_oidc_assertion_parse_template_file(rd_kafka_t *rk, + const char *jwt_template_file_path, + cJSON **header, + cJSON **payload) { + char *template_content = NULL; + cJSON *template_json = NULL; + int ret = -1; + size_t file_size; + + *header = NULL; + *payload = NULL; + + template_content = + rd_file_read(jwt_template_file_path, &file_size, 1024 * 1024); + if (!template_content) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to open JWT template file: %s", + jwt_template_file_path); + return -1; + } + + if (file_size == 0) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "JWT template file is empty or invalid"); + rd_free(template_content); + return -1; + } + + template_json = cJSON_Parse((char *)template_content); + if (!template_json) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to parse JWT template JSON"); + goto cleanup; + } + + cJSON *header_item = cJSON_GetObjectItem(template_json, "header"); + cJSON *payload_item = cJSON_GetObjectItem(template_json, "payload"); + + if (!header_item || !payload_item) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "JWT template must contain both 'header' " + "and 'payload' objects"); + goto cleanup; + } + + *header = cJSON_Duplicate(header_item, 1); + *payload = cJSON_Duplicate(payload_item, 1); + + if (!*header || !*payload) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to duplicate header or payload objects"); + if (*header) { + cJSON_Delete(*header); + *header = NULL; + } + goto cleanup; + } + + ret = 0; + +cleanup: + if (template_content) + rd_free(template_content); + if (template_json) + cJSON_Delete(template_json); + + return ret; +} + +/** + * @brief Create JWT assertion. + * + * Creates a JWT token signed with the specified private key using the + * algorithm specified. The token can be created from a template file or + * will create a minimal default token if no template is provided. + * + * @param rk The rd_kafka_t instance for logging + * @param private_key_pem PEM formatted private key string (mutually exclusive + * with key_file_location) + * @param key_file_location Path to private key file (mutually exclusive with + * private_key_pem) + * @param passphrase Optional passphrase for encrypted private key + * @param token_signing_algo Algorithm to use for signing (RS256 or ES256) + * @param jwt_template_file Optional path to JWT template file + * @param subject Optional subject claim value. + * @param issuer Optional issuer claim value. + * @param audience Optional audience claim value. + * @param nbf `nbf` claim value to express seconds of validity in the past. + * @param exp `exp` claim value to express seconds of validity in the future. + * @param jti_include Whether to include a JTI claim (UUID) + * + * @returns Newly allocated JWT string, caller must free with rd_free(). NULL on + * error. + * + * @locality Any thread. + */ +static char *rd_kafka_oidc_assertion_create( + rd_kafka_t *rk, + const char *private_key_pem, + const char *key_file_location, + const char *passphrase, + const rd_kafka_oauthbearer_assertion_algorithm_t token_signing_algo, + const char *jwt_template_file, + const char *subject, + const char *issuer, + const char *audience, + const int nbf, + const int exp, + const rd_bool_t jti_include) { + + char *encoded_header = NULL; + char *encoded_payload = NULL; + char *encoded_signature = NULL; + char *unsigned_token = NULL; + char *result = NULL; + char *header_str = NULL; + char *payload_str = NULL; + EVP_PKEY *pkey = NULL; + BIO *bio = NULL; + cJSON *header_json_obj = NULL; + cJSON *payload_json_obj = NULL; + EVP_MD_CTX *mdctx = NULL; + unsigned char *sig = NULL; + rd_chariov_t header_iov; + rd_chariov_t payload_iov; + rd_chariov_t sig_iov; + rd_kafka_Uuid_t jti_uuid; + char *jti_uuid_str = NULL; + + rd_ts_t issued_at = rd_uclock() / 1000000; + rd_ts_t not_before = issued_at - nbf; + rd_ts_t expiration_time = issued_at + exp; + + if (jwt_template_file) { + if (rd_kafka_oidc_assertion_parse_template_file( + rk, jwt_template_file, &header_json_obj, + &payload_json_obj) != 0) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to process JWT template file %s", + jwt_template_file); + return NULL; + } + } else { + header_json_obj = cJSON_CreateObject(); + payload_json_obj = cJSON_CreateObject(); + } + + /* Add required header fields */ + cJSON_DeleteItemFromObjectCaseSensitive(header_json_obj, "alg"); + cJSON_DeleteItemFromObjectCaseSensitive(header_json_obj, "typ"); + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, "iat"); + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, "exp"); + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, "nbf"); + cJSON_AddStringToObject( + header_json_obj, "alg", + rd_kafka_oidc_assertion_get_algo_label(token_signing_algo)); + cJSON_AddStringToObject(header_json_obj, "typ", "JWT"); + + /* Add required payload fields */ + cJSON_AddNumberToObject(payload_json_obj, "iat", (double)issued_at); + cJSON_AddNumberToObject(payload_json_obj, "exp", + (double)expiration_time); + cJSON_AddNumberToObject(payload_json_obj, "nbf", (double)not_before); + + if (subject) { + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, + "sub"); + cJSON_AddStringToObject(payload_json_obj, "sub", subject); + } + + if (issuer) { + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, + "iss"); + cJSON_AddStringToObject(payload_json_obj, "iss", issuer); + } + + if (audience) { + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, + "aud"); + cJSON_AddStringToObject(payload_json_obj, "aud", audience); + } + + if (jti_include) { + jti_uuid = rd_kafka_Uuid_random(); + jti_uuid_str = rd_kafka_Uuid_str(&jti_uuid); + cJSON_DeleteItemFromObjectCaseSensitive(payload_json_obj, + "jti"); + cJSON_AddStringToObject(payload_json_obj, "jti", jti_uuid_str); + rd_free(jti_uuid_str); + } + + header_str = cJSON_PrintUnformatted(header_json_obj); + payload_str = cJSON_PrintUnformatted(payload_json_obj); + + if (!header_str || !payload_str) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to convert template objects to JSON"); + goto cleanup; + } + + header_iov.ptr = header_str; + header_iov.size = strlen(header_str); + encoded_header = rd_base64_encode_str_urlsafe(&header_iov); + + payload_iov.ptr = payload_str; + payload_iov.size = strlen(payload_str); + encoded_payload = rd_base64_encode_str_urlsafe(&payload_iov); + if (!encoded_header || !encoded_payload) + goto cleanup; + + size_t unsigned_token_len = + strlen(encoded_header) + strlen(encoded_payload) + 2; + unsigned_token = rd_malloc(unsigned_token_len); + + if (!unsigned_token) + goto cleanup; + rd_snprintf(unsigned_token, unsigned_token_len, "%s.%s", encoded_header, + encoded_payload); + + if (private_key_pem) { + bio = BIO_new_mem_buf((void *)private_key_pem, -1); + } else if (key_file_location) { + bio = BIO_new_file(key_file_location, "r"); + } + + if (!bio) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to create BIO for private key"); + goto cleanup; + } + + if (passphrase) { + pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, + (void *)passphrase); + } else { + pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL); + } + BIO_free(bio); + bio = NULL; + + if (!pkey) { + rd_kafka_log(rk, LOG_ERR, "JWT", "Failed to load private key"); + goto cleanup; + } + + mdctx = EVP_MD_CTX_new(); + if (!mdctx) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to create message digest context"); + goto cleanup; + } + + const EVP_MD *md = EVP_sha256(); /* Both RS256 and ES256 use SHA-256 */ + + if (EVP_DigestSignInit(mdctx, NULL, md, NULL, pkey) != 1) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to initialize signing context"); + goto cleanup; + } + + if (EVP_DigestSignUpdate(mdctx, unsigned_token, + strlen(unsigned_token)) != 1) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to update digest with token data"); + goto cleanup; + } + + size_t siglen = 0; + if (EVP_DigestSignFinal(mdctx, NULL, &siglen) != 1) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to get signature length"); + goto cleanup; + } + + sig = rd_malloc(siglen); + if (!sig) { + rd_kafka_log(rk, LOG_ERR, "JWT", + "Failed to allocate memory for signature"); + goto cleanup; + } + + if (EVP_DigestSignFinal(mdctx, sig, &siglen) != 1) { + rd_kafka_log(rk, LOG_ERR, "JWT", "Failed to create signature"); + goto cleanup; + } + + sig_iov.ptr = (char *)sig; + sig_iov.size = siglen; + encoded_signature = rd_base64_encode_str_urlsafe(&sig_iov); + + if (!encoded_signature) + goto cleanup; + + size_t jwt_len = strlen(encoded_header) + strlen(encoded_payload) + + strlen(encoded_signature) + 3; + result = rd_malloc(jwt_len); + if (!result) + goto cleanup; + rd_snprintf(result, jwt_len, "%s.%s.%s", encoded_header, + encoded_payload, encoded_signature); + +cleanup: + if (encoded_header) + rd_free(encoded_header); + if (encoded_payload) + rd_free(encoded_payload); + if (encoded_signature) + rd_free(encoded_signature); + if (unsigned_token) + rd_free(unsigned_token); + if (sig) + rd_free(sig); + + if (header_json_obj) { + if (header_str) + free(header_str); /* cJSON_PrintUnformatted uses malloc + */ + cJSON_Delete(header_json_obj); + } else if (header_str) { + rd_free(header_str); /* rd_malloc was used */ + } + + if (payload_json_obj) { + if (payload_str) + free(payload_str); /* cJSON_PrintUnformatted uses malloc + */ + cJSON_Delete(payload_json_obj); + } else if (payload_str) { + rd_free(payload_str); /* rd_malloc was used */ + } + + if (pkey) + EVP_PKEY_free(pkey); + if (mdctx) + EVP_MD_CTX_free(mdctx); + + return result; +} + + +/** + * @brief Build request body for JWT bearer token request. + * + * Creates a URL-encoded request body for token exchange with the JWT assertion + * and optional scope. + * + * @param assertion The JWT assertion to include in the request. + * @param scope Optional scope to include in the request (will be URL encoded). + * + * @returns Newly allocated string with the URL-encoded request body. + * Caller must free with rd_free(). NULL on memory allocation failure. + * + * @locality Any thread. + */ +static char *rd_kafka_oidc_jwt_bearer_build_request_body(const char *assertion, + const char *scope) { + const char *assertion_prefix = + "grant_type=urn:ietf:params:oauth:" + "grant-type:jwt-bearer" + "&assertion="; + int assertion_prefix_len = strlen(assertion_prefix) + strlen(assertion); + int body_size = assertion_prefix_len + 1; + char *scope_escaped = NULL; + if (scope) { + scope_escaped = curl_easy_escape(NULL, scope, 0); + body_size += strlen("&scope=") + strlen(scope_escaped); + } + + char *body = rd_malloc(body_size); + + rd_snprintf(body, body_size, "%s%s", assertion_prefix, assertion); + if (scope) { + rd_snprintf(&body[assertion_prefix_len], + body_size - assertion_prefix_len, "&scope=%s", + scope_escaped); + rd_free(scope_escaped); + } + return body; +} + +/** + * @brief JWT assertion from file + * + * @param file_path Path to the file containing the JWT assertion. + * + * @returns Newly allocated string with the JWT assertion. + * Caller must free with rd_free(). NULL on error. + */ +static char *rd_kafka_oidc_assertion_read_from_file(const char *file_path) { + if (!file_path) + return NULL; + const size_t max_size = 1024 * 1024; /* 1MB limit */ + return rd_file_read(file_path, NULL, max_size); +} + +/** + * @brief Try to validate a token field from the JSON response. + * Extracts and validates the token, then decodes its payload to get + * subject and expiration. + * + * @param rk The rd_kafka_t instance + * @param json The JSON response from the token endpoint + * @param field_name The name of the field to extract (e.g., "access_token" or + * "id_token") + * @param token_out Pointer to store the extracted token + * @param sub_out Pointer to store the subject from the token + * @param exp_out Pointer to store the expiration from the token + * @param errstr_out Buffer to store error message + * @param errstr_size Size of error message buffer + * + * @returns The extracted token or NULL on failure. + */ +static char *rd_kafka_oidc_token_try_validate(cJSON *json, + const char *field, + const char *sub_claim_name, + char **sub, + double *exp, + char *errstr, + size_t errstr_size) { + cJSON *access_token_json, *jwt_exp, *jwt_sub, *payloads = NULL; + char *jwt_token = NULL, *decoded_payloads = NULL; + const char *decode_errstr = NULL; + *sub = NULL; + + access_token_json = cJSON_GetObjectItem(json, field); + + if (!access_token_json) { + rd_snprintf(errstr, errstr_size, + "Expected JSON response with \"%s\" field", field); + goto fail; + } + + jwt_token = cJSON_GetStringValue(access_token_json); + if (!jwt_token) { + rd_snprintf(errstr, errstr_size, + "Expected token as a string value"); + goto fail; + } + + decode_errstr = + rd_kafka_jwt_b64_decode_payload(jwt_token, &decoded_payloads); + if (decode_errstr != NULL) { + rd_snprintf(errstr, errstr_size, + "Failed to decode JWT payload: %s", decode_errstr); + goto fail; + } + + payloads = cJSON_Parse(decoded_payloads); + if (payloads == NULL) { + rd_snprintf(errstr, errstr_size, + "Failed to parse JSON JWT payload"); + goto fail; + } + + jwt_exp = cJSON_GetObjectItem(payloads, "exp"); + if (jwt_exp == NULL) { + rd_snprintf(errstr, errstr_size, + "Expected JSON JWT response with " + "\"exp\" field"); + goto fail; + } + + *exp = cJSON_GetNumberValue(jwt_exp); + if (*exp <= 0) { + rd_snprintf(errstr, errstr_size, + "Expected JSON JWT response with " + "valid \"exp\" field"); + goto fail; + } + + rd_dassert(sub_claim_name && *sub_claim_name); + + jwt_sub = cJSON_GetObjectItem(payloads, sub_claim_name); + if (jwt_sub == NULL) { + rd_snprintf(errstr, errstr_size, + "Expected JSON JWT response with " + "\"%s\" field", + sub_claim_name); + goto fail; + } + + *sub = cJSON_GetStringValue(jwt_sub); + if (*sub == NULL || **sub == '\0') { + /* Reset to NULL to prevent a dangling pointer to cJSON + * internal memory after cJSON_Delete(payloads) */ + *sub = NULL; + rd_snprintf(errstr, errstr_size, + "Expected JSON JWT response with " + "valid \"%s\" field (non-empty value required)", + sub_claim_name); + goto fail; + } + *sub = rd_strdup(*sub); +done: + if (payloads) + cJSON_Delete(payloads); + if (decoded_payloads) + rd_free(decoded_payloads); + return jwt_token; +fail: + jwt_token = NULL; + goto done; +} + +/** + * @brief Implementation of JWT token refresh callback function. + * Creates a JWT assertion, exchanges it for an access token, + * and sets the token for SASL OAUTHBEARER authentication. + * + * @param rk The rd_kafka_t instance. + * @param oauthbearer_config The OAUTHBEARER configuration. + * @param opaque Opaque pointer passed to the callback. + * + * @locality rdkafka main thread + */ +void rd_kafka_oidc_token_jwt_bearer_refresh_cb(rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque) { + const int timeout_s = 20; + const int retry = 4; + const int retry_ms = 5 * 1000; + + char *jwt_assertion = NULL; + char *request_body = NULL; + struct curl_slist *headers = NULL; + rd_http_error_t *herr = NULL; + cJSON *json = NULL; + char *jwt_token = NULL; + char set_token_errstr[512]; + double exp = 0; + char **extensions = NULL; + char **extension_key_value = NULL; + size_t extension_key_value_cnt = 0; + size_t extension_cnt; + char *sub = NULL; + char validate_errstr[512]; + + if (rd_kafka_terminating(rk)) + return; + + if (rk->rk_conf.sasl.oauthbearer.assertion.file) { + jwt_assertion = rd_kafka_oidc_assertion_read_from_file( + rk->rk_conf.sasl.oauthbearer.assertion.file); + } else { + jwt_assertion = rd_kafka_oidc_assertion_create( + rk, rk->rk_conf.sasl.oauthbearer.assertion.private_key.pem, + rk->rk_conf.sasl.oauthbearer.assertion.private_key.file, + rk->rk_conf.sasl.oauthbearer.assertion.private_key + .passphrase, + rk->rk_conf.sasl.oauthbearer.assertion.algorithm, + rk->rk_conf.sasl.oauthbearer.assertion.jwt_template_file, + rk->rk_conf.sasl.oauthbearer.assertion.claim.subject, + rk->rk_conf.sasl.oauthbearer.assertion.claim.issuer, + rk->rk_conf.sasl.oauthbearer.assertion.claim.audience, + rk->rk_conf.sasl.oauthbearer.assertion.claim.not_before_s, + rk->rk_conf.sasl.oauthbearer.assertion.claim.expiration_s, + rk->rk_conf.sasl.oauthbearer.assertion.claim.jti_include); + } + + if (!jwt_assertion) { + rd_kafka_oauthbearer_set_token_failure( + rk, "Failed to create JWT assertion"); + goto done; + } + + request_body = rd_kafka_oidc_jwt_bearer_build_request_body( + jwt_assertion, rk->rk_conf.sasl.oauthbearer.scope); + + if (!request_body) { + rd_kafka_oauthbearer_set_token_failure( + rk, "Failed to build JWT request body"); + goto done; + } + + headers = curl_slist_append( + headers, "Content-Type: application/x-www-form-urlencoded"); + headers = curl_slist_append(headers, "Accept: application/json"); + + herr = rd_http_post_expect_json( + rk, rk->rk_conf.sasl.oauthbearer.token_endpoint_url, headers, + request_body, strlen(request_body), timeout_s, retry, retry_ms, + &json); + + if (unlikely(herr != NULL)) { + rd_kafka_log( + rk, LOG_ERR, "JWT", + "Failed to retrieve JWT token from \"%s\": %s (%d)", + rk->rk_conf.sasl.oauthbearer.token_endpoint_url, + herr->errstr, herr->code); + rd_kafka_oauthbearer_set_token_failure(rk, herr->errstr); + rd_http_error_destroy(herr); + goto done; + } + + /* + * RFC 7523 Section 1 says that an access token should be returned + * https://datatracker.ietf.org/doc/html/rfc7523#section-1 + * Some providers (e.g. GCP) return an `id_token` instead, depending + * on the configured `target_audience` in the request JWT bearer token. + * This may be because the validation endpoint is not accessible + * for validating the `access_token` while the `id_token` is validated + * through the JWKS URL. + * This function will try to validate the `access_token` and then the + * `id_token`. + */ + jwt_token = rd_kafka_oidc_token_try_validate( + json, "access_token", rk->rk_conf.sasl.oauthbearer.sub_claim_name, + &sub, &exp, validate_errstr, sizeof(validate_errstr)); + if (!jwt_token) + jwt_token = rd_kafka_oidc_token_try_validate( + json, "id_token", + rk->rk_conf.sasl.oauthbearer.sub_claim_name, &sub, &exp, + validate_errstr, sizeof(validate_errstr)); + + if (!jwt_token) { + rd_kafka_oauthbearer_set_token_failure(rk, validate_errstr); + goto done; + } + + + if (rk->rk_conf.sasl.oauthbearer.extensions_str) { + extensions = + rd_string_split(rk->rk_conf.sasl.oauthbearer.extensions_str, + ',', rd_true, &extension_cnt); + + extension_key_value = rd_kafka_conf_kv_split( + (const char **)extensions, extension_cnt, + &extension_key_value_cnt); + } + + if (rd_kafka_oauthbearer_set_token( + rk, jwt_token, (int64_t)exp * 1000, sub, + (const char **)extension_key_value, extension_key_value_cnt, + set_token_errstr, + sizeof(set_token_errstr)) != RD_KAFKA_RESP_ERR_NO_ERROR) { + rd_kafka_oauthbearer_set_token_failure(rk, validate_errstr); + } + +done: + RD_IF_FREE(sub, rd_free); + RD_IF_FREE(jwt_assertion, rd_free); + RD_IF_FREE(request_body, rd_free); + RD_IF_FREE(headers, curl_slist_free_all); + RD_IF_FREE(json, cJSON_Delete); + RD_IF_FREE(extensions, rd_free); + RD_IF_FREE(extension_key_value, rd_free); + /* jwt_token is freed as part of the json object */ +} + +/** + * @brief Implementation of Oauth/OIDC token refresh callback function, + * will receive the JSON response after HTTP call to token provider, + * then extract the jwt from the JSON response, and forward it to + * the broker. + */ +void rd_kafka_oidc_token_client_credentials_refresh_cb( + rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque) { + const int timeout_s = 20; + const int retry = 4; + const int retry_ms = 5 * 1000; + + double exp; + + cJSON *json = NULL; + + rd_http_error_t *herr; + + char *jwt_token; + char *post_fields = NULL; + + struct curl_slist *headers = NULL; + + const char *token_url; + char *sub = NULL; + + size_t post_fields_size; + size_t extension_cnt; + size_t extension_key_value_cnt = 0; + + char set_token_errstr[512]; + + char **extensions = NULL; + char **extension_key_value = NULL; + + if (rd_kafka_terminating(rk)) + return; + + rd_kafka_oidc_client_credentials_build_headers( + rk->rk_conf.sasl.oauthbearer.client_id, + rk->rk_conf.sasl.oauthbearer.client_secret, &headers); + + /* Build post fields */ + rd_kafka_oidc_client_credentials_build_post_fields( + rk->rk_conf.sasl.oauthbearer.scope, &post_fields, + &post_fields_size); + + token_url = rk->rk_conf.sasl.oauthbearer.token_endpoint_url; + + herr = rd_http_post_expect_json(rk, token_url, headers, post_fields, + post_fields_size, timeout_s, retry, + retry_ms, &json); + + if (unlikely(herr != NULL)) { + rd_kafka_log(rk, LOG_ERR, "OIDC", + "Failed to retrieve OIDC " + "token from \"%s\": %s (%d)", + token_url, herr->errstr, herr->code); + rd_kafka_oauthbearer_set_token_failure(rk, herr->errstr); + rd_http_error_destroy(herr); + goto done; + } + + jwt_token = rd_kafka_oidc_token_try_validate( + json, "access_token", rk->rk_conf.sasl.oauthbearer.sub_claim_name, + &sub, &exp, set_token_errstr, sizeof(set_token_errstr)); + if (!jwt_token) { + rd_kafka_oauthbearer_set_token_failure(rk, set_token_errstr); + goto done; + } + + if (rk->rk_conf.sasl.oauthbearer.extensions_str) { + extensions = + rd_string_split(rk->rk_conf.sasl.oauthbearer.extensions_str, + ',', rd_true, &extension_cnt); + + extension_key_value = rd_kafka_conf_kv_split( + (const char **)extensions, extension_cnt, + &extension_key_value_cnt); + } + + if (rd_kafka_oauthbearer_set_token( + rk, jwt_token, (int64_t)exp * 1000, sub, + (const char **)extension_key_value, extension_key_value_cnt, + set_token_errstr, + sizeof(set_token_errstr)) != RD_KAFKA_RESP_ERR_NO_ERROR) + rd_kafka_oauthbearer_set_token_failure(rk, set_token_errstr); + +done: + RD_IF_FREE(sub, rd_free); + RD_IF_FREE(post_fields, rd_free); + RD_IF_FREE(json, cJSON_Delete); + RD_IF_FREE(headers, curl_slist_free_all); + RD_IF_FREE(extensions, rd_free); + RD_IF_FREE(extension_key_value, rd_free); +} + +/** + * @brief Implementation of Oauth/OIDC token refresh callback function + * for Azure IMDS, + * will receive the JSON response after HTTP(S) GET call to token + * provider, then extract the jwt from the JSON response, and + * forward it to the broker. + */ +void rd_kafka_oidc_token_metadata_azure_imds_refresh_cb( + rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque) { + const int timeout_s = 20; + const int retries = 4; + const int retry_ms = 5 * 1000; + + double exp; + + cJSON *json = NULL; + + rd_http_error_t *herr; + + char *jwt_token; + + struct curl_slist *headers = NULL; + + const char *token_endpoint_url_initial = NULL; + char *token_endpoint_url = NULL; + char *sub = NULL; + + size_t extension_cnt; + size_t extension_key_value_cnt = 0; + + char set_token_errstr[512]; + + char **extensions = NULL; + char **extension_key_value = NULL; + char *query = NULL; + static char *headers_array[] = {"Metadata: true"}; + + if (rd_kafka_terminating(rk)) + return; + + if (rk->rk_conf.sasl.oauthbearer_config) + query = rd_kafka_conf_kv_get( + rk->rk_conf.sasl.oauthbearer_config, "query", ','); + token_endpoint_url_initial = + rk->rk_conf.sasl.oauthbearer.token_endpoint_url; + if (!token_endpoint_url_initial) + token_endpoint_url_initial = + RD_KAFKA_SASL_OAUTHBEARER_METADATA_AUTHENTICATION_URL_AZURE_IMDS; + if (query && *query) { + token_endpoint_url = rd_http_get_params_append( + token_endpoint_url_initial, query); + + if (token_endpoint_url == NULL) { + rd_snprintf( + set_token_errstr, sizeof(set_token_errstr), + "Failed to append params \"%s\" to token endpoint " + "URL \"%s\"", + query, + rk->rk_conf.sasl.oauthbearer.token_endpoint_url); + rd_kafka_log(rk, LOG_ERR, "OIDC", "%s", + set_token_errstr); + rd_kafka_oauthbearer_set_token_failure( + rk, set_token_errstr); + goto done; + } + } else { + token_endpoint_url = rd_strdup(token_endpoint_url_initial); + } + + herr = rd_http_get_json(rk, token_endpoint_url, headers_array, 1, + timeout_s, retries, retry_ms, &json); + + if (unlikely(herr != NULL)) { + rd_kafka_log(rk, LOG_ERR, "OIDC", + "Failed to retrieve OIDC " + "token from \"%s\": %s (%d)", + token_endpoint_url, herr->errstr, herr->code); + rd_kafka_oauthbearer_set_token_failure(rk, herr->errstr); + rd_http_error_destroy(herr); + goto done; + } + + jwt_token = rd_kafka_oidc_token_try_validate( + json, "access_token", rk->rk_conf.sasl.oauthbearer.sub_claim_name, + &sub, &exp, set_token_errstr, sizeof(set_token_errstr)); + if (!jwt_token) { + rd_kafka_oauthbearer_set_token_failure(rk, set_token_errstr); + goto done; + } + + if (rk->rk_conf.sasl.oauthbearer.extensions_str) { + extensions = + rd_string_split(rk->rk_conf.sasl.oauthbearer.extensions_str, + ',', rd_true, &extension_cnt); + + extension_key_value = rd_kafka_conf_kv_split( + (const char **)extensions, extension_cnt, + &extension_key_value_cnt); + } + + if (rd_kafka_oauthbearer_set_token( + rk, jwt_token, (int64_t)exp * 1000, sub, + (const char **)extension_key_value, extension_key_value_cnt, + set_token_errstr, + sizeof(set_token_errstr)) != RD_KAFKA_RESP_ERR_NO_ERROR) + rd_kafka_oauthbearer_set_token_failure(rk, set_token_errstr); + +done: + RD_IF_FREE(sub, rd_free); + RD_IF_FREE(json, cJSON_Delete); + RD_IF_FREE(headers, curl_slist_free_all); + RD_IF_FREE(extensions, rd_free); + RD_IF_FREE(extension_key_value, rd_free); + RD_IF_FREE(token_endpoint_url, rd_free); + RD_IF_FREE(query, rd_free); +} + +/** + * @brief OIDC token refresh callback for aws_iam metadata + * authentication. As of now, librdkafka does not implement AWS IAM + * natively. + */ +void rd_kafka_oidc_token_metadata_aws_iam_refresh_cb( + rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque) { + /* + * TODO: Implement native librdkafka flow. + * Currently, this block serves as a defensive check to reject this + * configuration until the feature is fully supported. + */ + rd_kafka_oauthbearer_set_token_failure( + rk, + "aws_iam OAUTHBEARER token retrieval is not implemented " + "natively in librdkafka; a token-refresh callback must supply " + "the token. High-level clients wire this automatically when " + "their AWS OAUTHBEARER package/extra is installed and " + "configured. Reaching this code path means method=oidc and " + "metadata.authentication.type=aws_iam were set but no " + "token-refresh callback is registered."); +} + +/** + * @brief Make sure the jwt is able to be extracted from HTTP(S) response. + * The JSON response after HTTP(S) call to token provider will be in + * rd_http_req_t.hreq_buf and jwt is the value of field "access_token", + * the format is {"access_token":"*******"}. + * This function mocks up the rd_http_req_t.hreq_buf using an dummy + * jwt. The rd_http_parse_json will extract the jwt from rd_http_req_t + * and make sure the extracted jwt is same with the dummy one. + */ +static int ut_sasl_oauthbearer_oidc_should_succeed(void) { + /* Generate a token in the https://jwt.io/ website by using the + * following steps: + * 1. Select the algorithm RS256 from the Algorithm drop-down menu. + * 2. Enter the header and the payload. + * payload should contains "exp", "iat", "sub", for example: + * payloads = {"exp": 1636532769, + "iat": 1516239022, + "sub": "sub"} + header should contains "kid", for example: + headers={"kid": "abcedfg"} */ + static const char *expected_jwt_token = + "eyJhbGciOiJIUzI1NiIsInR5" + "cCI6IkpXVCIsImtpZCI6ImFiY2VkZmcifQ" + "." + "eyJpYXQiOjE2MzIzNzUzMjAsInN1YiI6InN" + "1YiIsImV4cCI6MTYzMjM3NTYyMH0" + "." + "bT5oY8K-rS2gQ7Awc40844bK3zhzBhZb7sputErqQHY"; + char *expected_token_value; + size_t token_len; + rd_http_req_t hreq; + rd_http_error_t *herr; + cJSON *json = NULL; + char *token; + cJSON *parsed_token; + rd_kafka_t *rk = rd_calloc(1, sizeof(*rk)); + + RD_UT_BEGIN(); + + herr = rd_http_req_init(rk, &hreq, ""); + + RD_UT_ASSERT(!herr, + "Expected initialize to succeed, " + "but failed with error code: %d, error string: %s", + herr->code, herr->errstr); + + token_len = strlen("access_token") + strlen(expected_jwt_token) + 8; + + expected_token_value = rd_malloc(token_len); + rd_snprintf(expected_token_value, token_len, "{\"%s\":\"%s\"}", + "access_token", expected_jwt_token); + rd_buf_write(hreq.hreq_buf, expected_token_value, token_len); + + herr = rd_http_parse_json(&hreq, &json); + RD_UT_ASSERT(!herr, + "Failed to parse JSON token: error code: %d, " + "error string: %s", + herr->code, herr->errstr); + + RD_UT_ASSERT(json, "Expected non-empty json."); + + parsed_token = cJSON_GetObjectItem(json, "access_token"); + + RD_UT_ASSERT(parsed_token, "Expected access_token in JSON response."); + token = parsed_token->valuestring; + + RD_UT_ASSERT(!strcmp(expected_jwt_token, token), + "Incorrect token received: " + "expected=%s; received=%s", + expected_jwt_token, token); + + rd_free(expected_token_value); + rd_http_error_destroy(herr); + rd_http_req_destroy(&hreq); + cJSON_Delete(json); + rd_free(rk); + + RD_UT_PASS(); +} + + +/** + * @brief Make sure JSON doesn't include the "access_token" key, + * it will fail and return an empty token. + */ +static int ut_sasl_oauthbearer_oidc_with_empty_key(void) { + static const char *empty_token_format = "{}"; + size_t token_len; + rd_http_req_t hreq; + rd_http_error_t *herr; + cJSON *json = NULL; + cJSON *parsed_token; + rd_kafka_t *rk = rd_calloc(1, sizeof(*rk)); + + RD_UT_BEGIN(); + + herr = rd_http_req_init(rk, &hreq, ""); + RD_UT_ASSERT(!herr, + "Expected initialization to succeed, " + "but it failed with error code: %d, error string: %s", + herr->code, herr->errstr); + + token_len = strlen(empty_token_format); + + rd_buf_write(hreq.hreq_buf, empty_token_format, token_len); + + herr = rd_http_parse_json(&hreq, &json); + + RD_UT_ASSERT(!herr, + "Expected JSON token parsing to succeed, " + "but it failed with error code: %d, error string: %s", + herr->code, herr->errstr); + + RD_UT_ASSERT(json, "Expected non-empty json."); + + parsed_token = cJSON_GetObjectItem(json, "access_token"); + + RD_UT_ASSERT(!parsed_token, + "Did not expecte access_token in JSON response"); + + rd_http_req_destroy(&hreq); + rd_http_error_destroy(herr); + cJSON_Delete(json); + cJSON_Delete(parsed_token); + rd_free(rk); + RD_UT_PASS(); +} + +/** + * @brief Make sure the post_fields return correct with the scope. + */ +static int ut_sasl_oauthbearer_oidc_post_fields(void) { + static const char *scope = "test-scope"; + static const char *expected_post_fields = + "grant_type=client_credentials&scope=test-scope"; + + size_t expected_post_fields_size = strlen(expected_post_fields); + + size_t post_fields_size; + + char *post_fields; + + RD_UT_BEGIN(); + + rd_kafka_oidc_client_credentials_build_post_fields(scope, &post_fields, + &post_fields_size); + + RD_UT_ASSERT(expected_post_fields_size == post_fields_size, + "Expected expected_post_fields_size is %" PRIusz + " received post_fields_size is %" PRIusz, + expected_post_fields_size, post_fields_size); + RD_UT_ASSERT(!strcmp(expected_post_fields, post_fields), + "Expected expected_post_fields is %s" + " received post_fields is %s", + expected_post_fields, post_fields); + + rd_free(post_fields); + + RD_UT_PASS(); +} + +/** + * @brief Make sure the post_fields return correct with the empty scope. + */ +static int ut_sasl_oauthbearer_oidc_post_fields_with_empty_scope(void) { + static const char *scope = NULL; + static const char *expected_post_fields = + "grant_type=client_credentials"; + + size_t expected_post_fields_size = strlen(expected_post_fields); + + size_t post_fields_size; + + char *post_fields; + + RD_UT_BEGIN(); + + rd_kafka_oidc_client_credentials_build_post_fields(scope, &post_fields, + &post_fields_size); + + RD_UT_ASSERT(expected_post_fields_size == post_fields_size, + "Expected expected_post_fields_size is %" PRIusz + " received post_fields_size is %" PRIusz, + expected_post_fields_size, post_fields_size); + RD_UT_ASSERT(!strcmp(expected_post_fields, post_fields), + "Expected expected_post_fields is %s" + " received post_fields is %s", + expected_post_fields, post_fields); + + rd_free(post_fields); + + RD_UT_PASS(); +} + +/** All JWTs use a fake signature since token_try_validate() only + * decodes and inspects the payload; signature verification is done + * separately by the broker. + * + * JWT payloads (decoded): + * JWT_SUB_ONLY: + * {"exp":9999999999,"iat":1000000000,"sub":"subject"} + * + * JWT_MULTI_CLAIMS: + * {"exp":9999999999,"iat":1000000000,"sub":"subject", + * "client_id":"client_id_123","azp":"azp_123"} + * + * JWT_EMPTY_SUB: + * {"exp":9999999999,"iat":1000000000,"sub":""} + * + * JWT_MISSING_SUB: + * {"exp":9999999999,"iat":1000000000,"client_id":"client_id_123"} + */ +/* payload: {"exp":9999999999,"iat":1000000000,"sub":"subject"} */ +#define UT_JWT_SUB_ONLY \ + "eyJhbGciOiJSUzI1NiIsImtpZCI6ImFiY2RlZmcifQ" \ + "." \ + "eyJleHAiOjk5OTk5OTk5OTksImlhdCI6MTAwMDAwMDAwMCwic" \ + "3ViIjoic3ViamVjdCJ9" \ + "." \ + "fakesignature" +/* payload: {"exp":9999999999,"iat":1000000000,"sub":"subject", + * "client_id":"client_id_123","azp":"azp_123"} */ +#define UT_JWT_MULTI_CLAIMS \ + "eyJhbGciOiJSUzI1NiIsImtpZCI6ImFiY2RlZmcifQ" \ + "." \ + "eyJleHAiOjk5OTk5OTk5OTksImlhdCI6MTAwMDAwMDAwMCwic3ViIjoic3ViamVj" \ + "dCIsImNsaWVudF9pZCI6ImNsaWVudF9pZF8xMjMiLCJhenAiOiJhenBfMTIzIn0" \ + "." \ + "fakesignature" +/* payload: {"exp":9999999999,"iat":1000000000,"sub":""} */ +#define UT_JWT_EMPTY_SUB \ + "eyJhbGciOiJSUzI1NiIsImtpZCI6ImFiY2RlZmcifQ" \ + "." \ + "eyJleHAiOjk5OTk5OTk5OTksImlhdCI6MTAwMDAwMDAwMCwic3ViIjoiIn0" \ + "." \ + "fakesignature" +/* payload: {"exp":9999999999,"iat":1000000000,"client_id":"client_id_123"} */ +#define UT_JWT_MISSING_SUB \ + "eyJhbGciOiJSUzI1NiIsImtpZCI6ImFiY2RlZmcifQ" \ + "." \ + "eyJleHAiOjk5OTk5OTk5OTksImlhdCI6MTAwMDAwMDAwMCwiY2xpZW50X2lkIjoi" \ + "Y2xpZW50X2lkXzEyMyJ9" \ + "." \ + "fakesignature" + +/** + * @brief Verifies the extraction logic of the subject from the configured JWT + * claim name, falls back to "sub" when unconfigured, and rejects missing or + * empty claim values. + */ +static int ut_sasl_oauthbearer_oidc_sub_claim_name(void) { + + const struct { + const char *test_name; + const char *jwt; + const char *sub_claim_name; + rd_bool_t expect_success; + const char *expected_sub; + } tests[] = { + {"Explicit 'sub' claim name", UT_JWT_SUB_ONLY, "sub", rd_true, + "subject"}, + {"Custom 'client_id' claim", UT_JWT_MULTI_CLAIMS, "client_id", + rd_true, "client_id_123"}, + {"Custom 'azp' claim", UT_JWT_MULTI_CLAIMS, "azp", rd_true, + "azp_123"}, + {"Custom 'client_id' claim succeeds without sub", + UT_JWT_MISSING_SUB, "client_id", rd_true, "client_id_123"}, + {"Missing 'sub' claim fails", UT_JWT_MISSING_SUB, "sub", rd_false, + NULL}, + {"Empty 'sub' value fails", UT_JWT_EMPTY_SUB, "sub", rd_false, + NULL}, + {"Nonexistent claim name fails", UT_JWT_SUB_ONLY, "nonexistent", + rd_false, NULL}, + }; + + unsigned int i; + + RD_UT_BEGIN(); + + for (i = 0; i < RD_ARRAYSIZE(tests); i++) { + char *sub = NULL; + double exp_v = 0; + char errstr[256]; + char *result; + cJSON *json; + char access_token_json[2048]; + + rd_snprintf(access_token_json, sizeof(access_token_json), + "{\"access_token\":\"%s\"}", tests[i].jwt); + json = cJSON_Parse(access_token_json); + RD_UT_ASSERT(json != NULL, "[%s] Failed to build test JSON", + tests[i].test_name); + + result = rd_kafka_oidc_token_try_validate( + json, "access_token", tests[i].sub_claim_name, &sub, &exp_v, + errstr, sizeof(errstr)); + + if (tests[i].expect_success) { + RD_UT_ASSERT(result != NULL, + "[%s] Expected success but got error: %s", + tests[i].test_name, errstr); + RD_UT_ASSERT(sub != NULL, "[%s] Expected non-NULL sub", + tests[i].test_name); + RD_UT_ASSERT(!strcmp(sub, tests[i].expected_sub), + "[%s] Expected sub '%s', got '%s'", + tests[i].test_name, tests[i].expected_sub, + sub); + } else { + RD_UT_ASSERT(result == NULL, + "[%s] Expected failure but got sub '%s'", + tests[i].test_name, sub ? sub : "(null)"); + } + + RD_IF_FREE(sub, rd_free); + cJSON_Delete(json); + } + + RD_UT_PASS(); +} + + +/** + * @brief make sure the jwt is able to be extracted from HTTP(S) requests + * or fail as expected. + */ +int unittest_sasl_oauthbearer_oidc(void) { + int fails = 0; + fails += ut_sasl_oauthbearer_oidc_should_succeed(); + fails += ut_sasl_oauthbearer_oidc_with_empty_key(); + fails += ut_sasl_oauthbearer_oidc_post_fields(); + fails += ut_sasl_oauthbearer_oidc_post_fields_with_empty_scope(); + fails += ut_sasl_oauthbearer_oidc_sub_claim_name(); + return fails; +} + +/** + * @brief Test the Base64Url encoding functionality. + * Verifies that the encoding correctly handles special characters + * and padding removal. + */ +static int ut_sasl_oauthbearer_oidc_jwt_bearer_base64url_encode(void) { + /* Test cases with expected inputs and outputs */ + static const struct { + const char *input; + const char *expected_output; + } test_cases[] = { + /* Regular case */ + {"Hello, world!", "SGVsbG8sIHdvcmxkIQ"}, + /* Case with padding characters that should be removed */ + {"test", "dGVzdA"}, + /* Empty string */ + {"", ""}, + /* Special characters that trigger Base64 padding */ + {"f", "Zg"}, + {"fo", "Zm8"}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg"}, + {"fooba", "Zm9vYmE"}, + {"foobar", "Zm9vYmFy"}, + /* Characters that produce + and / in standard Base64 */ + {"\x3E\x3F", + "Pj8"}, /* encodes to ">?" in standard Base64 with + and / */ + }; + unsigned int i; + + RD_UT_BEGIN(); + + for (i = 0; i < RD_ARRAYSIZE(test_cases); i++) { + rd_chariov_t input_iov; + input_iov.ptr = (char *)test_cases[i].input; + input_iov.size = strlen(test_cases[i].input); + char *output = rd_base64_encode_str_urlsafe(&input_iov); + + RD_UT_ASSERT(output != NULL, + "Expected non-NULL output for input: %s", + test_cases[i].input); + + RD_UT_ASSERT(!strcmp(output, test_cases[i].expected_output), + "Base64Url encoding failed: expected %s, got %s", + test_cases[i].expected_output, output); + + rd_free(output); + } + + RD_UT_PASS(); +} + +/** + * @brief Test JWT request body building. + * Verifies that the request body is correctly formatted with + * the required parameters. + */ +static int ut_sasl_oauthbearer_oidc_jwt_bearer_build_request_body(void) { + const char *assertion = "test.jwt.assertion"; + const char *scope = "test.scope"; + const char *expected = + "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=" + "test.jwt.assertion&scope=test.scope"; + char *body; + + RD_UT_BEGIN(); + + body = rd_kafka_oidc_jwt_bearer_build_request_body(assertion, scope); + + RD_UT_ASSERT(body != NULL, "Expected non-NULL request body"); + + RD_UT_ASSERT(!strcmp(body, expected), + "Request body incorrect: expected '%s', got '%s'", + expected, body); + + rd_free(body); + + RD_UT_PASS(); +} + +/** + * @brief Test JWT assertion file parsing. + * Verifies that the function correctly reads a JWT from a file. + */ +static int ut_sasl_oauthbearer_oidc_assertion_parse_from_file(void) { + + char tempfile_path[512]; + FILE *tempfile; + const char *test_jwt = "header.payload.signature"; + char *result; + + RD_UT_BEGIN(); + + tempfile = rd_file_mkstemp("rdtmp", "wb", tempfile_path, + sizeof(tempfile_path)); + fprintf(tempfile, "%s", test_jwt); + fclose(tempfile); + + /* Test parsing from file */ + result = rd_kafka_oidc_assertion_read_from_file(tempfile_path); + RD_UT_ASSERT(result != NULL, + "Expected non-NULL result from parsing file"); + RD_UT_ASSERT(!strcmp(result, test_jwt), + "Incorrect JWT parsed: expected '%s', got '%s'", test_jwt, + result); + + rd_free(result); + + /* Test with NULL path */ + result = rd_kafka_oidc_assertion_read_from_file(NULL); + RD_UT_ASSERT(result == NULL, "Expected NULL result with NULL path"); + + /* Test with non-existent file */ + result = + rd_kafka_oidc_assertion_read_from_file("/non/existent/file/path"); + RD_UT_ASSERT(result == NULL, + "Expected NULL result with non-existent file"); + + remove(tempfile_path); + + RD_UT_PASS(); +} + +/** + * @brief Mock function for testing JWT template processing. + * Creates a file with valid JWT template JSON. + */ +static char *ut_create_mock_jwt_template_file(void) { + FILE *tempfile; + char tempfile_path[512]; + + const char *template_json = + "{\n" + " \"header\": {\n" + " \"kid\": \"test-key-id\"\n" + " },\n" + " \"payload\": {\n" + " \"sub\": \"test-subject\",\n" + " \"aud\": \"test-audience\"\n" + " }\n" + "}"; + + tempfile = rd_file_mkstemp("rdtmp", "wb", tempfile_path, + sizeof(tempfile_path)); + if (!tempfile) + return NULL; + + fprintf(tempfile, "%s", template_json); + fclose(tempfile); + + return rd_strdup(tempfile_path); +} + +/** + * @brief Test JWT template file processing. + * Verifies that the function correctly parses header and payload from + * template. + */ +static int ut_sasl_oauthbearer_oidc_assertion_process_template_file(void) { + char *template_path; + rd_kafka_t *rk; + cJSON *header = NULL, *payload = NULL; + int result; + + RD_UT_BEGIN(); + + rk = rd_calloc(1, sizeof(*rk)); + + template_path = ut_create_mock_jwt_template_file(); + RD_UT_ASSERT(template_path != NULL, "Failed to create template file"); + + /* Test template processing */ + result = rd_kafka_oidc_assertion_parse_template_file(rk, template_path, + &header, &payload); + RD_UT_ASSERT(result == 0, "Expected success from template processing"); + RD_UT_ASSERT(header != NULL, "Expected non-NULL header JSON"); + RD_UT_ASSERT(payload != NULL, "Expected non-NULL payload JSON"); + + /* Verify header contents */ + cJSON *kid = cJSON_GetObjectItem(header, "kid"); + RD_UT_ASSERT(kid != NULL, "Expected kid in header"); + RD_UT_ASSERT(cJSON_IsString(kid), "Expected kid to be string"); + RD_UT_ASSERT(!strcmp(cJSON_GetStringValue(kid), "test-key-id"), + "Incorrect kid value"); + + /* Verify payload contents */ + cJSON *sub = cJSON_GetObjectItem(payload, "sub"); + RD_UT_ASSERT(sub != NULL, "Expected sub in payload"); + RD_UT_ASSERT(cJSON_IsString(sub), "Expected sub to be string"); + RD_UT_ASSERT(!strcmp(cJSON_GetStringValue(sub), "test-subject"), + "Incorrect sub value"); + + cJSON *aud = cJSON_GetObjectItem(payload, "aud"); + RD_UT_ASSERT(aud != NULL, "Expected aud in payload"); + RD_UT_ASSERT(cJSON_IsString(aud), "Expected aud to be string"); + RD_UT_ASSERT(!strcmp(cJSON_GetStringValue(aud), "test-audience"), + "Incorrect aud value"); + + /* Test with non-existent file */ + cJSON_Delete(header); + cJSON_Delete(payload); + header = NULL; + payload = NULL; + + result = rd_kafka_oidc_assertion_parse_template_file( + rk, "/non/existent/file", &header, &payload); + RD_UT_ASSERT(result == -1, "Expected failure with non-existent file"); + RD_UT_ASSERT(header == NULL, + "Expected NULL header with failed processing"); + RD_UT_ASSERT(payload == NULL, + "Expected NULL payload with failed processing"); + + unlink(template_path); + rd_free(template_path); + rd_free(rk); + if (header) + cJSON_Delete(header); + if (payload) + cJSON_Delete(payload); + + RD_UT_PASS(); +} + +/** + * @brief Test JWT assertion creation with minimal approach. + * Creates a simplified test that validates the format of the created + * JWT. + */ +static int ut_sasl_oauthbearer_oidc_assertion_create(void) { + rd_kafka_t *rk; + char *private_key_pem; + char *jwt; + char *header_part, *payload_part, *signature_part; + char *dot1, *dot2; + + RD_UT_BEGIN(); + + rk = rd_calloc(1, sizeof(*rk)); + + /* Random key for signing */ + private_key_pem = + "-----BEGIN PRIVATE KEY-----\n" + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCuBS7qG5Cd2voa\n" + "7nSU2xaDbe6QOYU2P4bIY58SKHbFyq1iB517r61ImsWD+UfZuVxCqXRaWdxxnG/D\n" + "5VGTQzBOZYlgSYxdJ1KvITXO8kj5i2zBT/LI9R9MTQ7nLFh+vQm1aM8Ts1PmA5t9\n" + "zFtR9B8RfqN9kbt+2LnLY57aJxEkFC3D89D0WWT97UJWKo7/vxMqp9K9uAIL2Efo\n" + "5rp9qwyPbx9LmTbfZ8Vog6mG6tAQQHSUqw0PnfhADCVCkYtkzYcyDZy3qZQFu1bY\n" + "KuuMoMjssyCUL5tTHyNZju0p3Z0bSfOV/nkqHpSSjHKCeQkSKS18/7In6cfY/M4k\n" + "8rM4HWkdAgMBAAECggEAFsTo2YrXxj/Dn8h5ioyMCpBUuZw9GNcBDLE0PAz9VW3q\n" + "d7wlV+ypkKlnlJgGVa+SKcrARZ4iYN8mJIyZutn8tRVF/0pASmP9xppizvwWnkgm\n" + "57hNPQwNl08x1v+PaK3VWl4nUh2RqbPpIXGetT9q3UAjpiduT++Nh9Y2D7cy3/Ro\n" + "ritnpBDs1R6y5J3rxiE1s8kLYwhDRCPsgUg/ZtKPDTTFz42ArrFeqM91FmjHYP3t\n" + "p9Uh6CIZ80D6CsMX/TnZFfhKe6EvKBSl4W6tcdFlnXW52fm/670iKSmcJ09+fzPO\n" + "T1BLrkXGv51bFnlvUyJqQGVEv5+0+HUX/oTpTknMQQKBgQDbYhqip5e8r1f5v32B\n" + "k1r3xtEiWU2mZoTHJu6bVeuigzVhz4pTMVZChElJ4QnhwwO0t5Oe4Su1MZtjMRw7\n" + "qIE+YM2pXss25LRXbmWItuRWINzpe8omlxQSOj2tNO/67l0P4vmmrT5wkU2cG6TR\n" + "ddzorO3NDA4MY4+Xdli+SHXwUQKBgQDLEMqlwyvaGjuZ30l6F13fWnEt9PNCtJsa\n" + "nsdKJKyFMThdysY/PK40o2GTRRhgYa2jigN3OCYSSznRRZRlqznqL1bOLlYV6zS8\n" + "TGhdLXuApyLAjZYIK4RtZJYGR9+yg8rH13uNektgW8KnHh5Ko/ptRVoEukf3SBsh\n" + "f0Fib3ylDQKBgE11Bth0+bMJ6bLpNEPiphSjosVQ6ISe37R8/3Pi0y5uyxM8tqcG\n" + "3WDg2gt2pAmM1CsjQcCv2cHAwQ81kLVTmkZO4W4yZOd9ulrARKMPh/EM61KYfVhA\n" + "sTp6S7py3WQocr0gM2rw8gHGm7NJY1j9F0EjhVaHMhKXuGQOyehtJw7xAoGAPwuA\n" + "jwRQSg+Y74XmbxRwHZcbynPhTpV6DkK7huZp9ZQ5ds0szZdOUqNi+PEbx1isKzj/\n" + "KHVzRHy8f5+FmicV/QIjhjHWokl6/vcN89faHzBE1tleejzgiYIQHfUUm3zVaUQa\n" + "ZOtSGaGDhpUQPIY6itBcSVl4XGqzmavDpgcNAMUCgYBFFGtG+RbSySzKfRUp3vc5\n" + "8YqIdrtXfW9gc9s1+Pw8wfgrY0Rrvy+e3ClSwgGENxgxBvWvhzq2m0S8x2jdLAl1\n" + "b+VLGCOpUvS4iN2yrHkoHS7BSW40wLuVooJUAaNOIEPqiv1JC75q2dhTRrANp6WB\n" + "bm+7yWVTNlXYuKQqtuOkNQ==\n" + "-----END PRIVATE KEY-----\n"; + + jwt = rd_kafka_oidc_assertion_create( + rk, private_key_pem, NULL, NULL, + RD_KAFKA_SASL_OAUTHBEARER_ASSERTION_ALGORITHM_RS256, NULL, + "test-subject", "test-issuer", "test-audience", 2, 300, rd_true); + + RD_UT_ASSERT(jwt != NULL, "Failed to create JWT assertion"); + + dot1 = strchr(jwt, '.'); + RD_UT_ASSERT(dot1 != NULL, "JWT missing first dot separator"); + + dot2 = strchr(dot1 + 1, '.'); + RD_UT_ASSERT(dot2 != NULL, "JWT missing second dot separator"); + + header_part = rd_strndup(jwt, dot1 - jwt); + payload_part = rd_strndup(dot1 + 1, dot2 - (dot1 + 1)); + signature_part = rd_strdup(dot2 + 1); + + RD_UT_ASSERT(strlen(header_part) > 0, "JWT header part is empty"); + RD_UT_ASSERT(strlen(payload_part) > 0, "JWT payload part is empty"); + RD_UT_ASSERT(strlen(signature_part) > 0, "JWT signature part is empty"); + + RD_UT_ASSERT(!strchr(header_part, '='), + "JWT header contains padding character"); + RD_UT_ASSERT(!strchr(payload_part, '='), + "JWT payload contains padding character"); + RD_UT_ASSERT(!strchr(signature_part, '='), + "JWT signature contains padding character"); + + RD_UT_ASSERT(!strchr(header_part, '+'), + "JWT header contains '+' character"); + RD_UT_ASSERT(!strchr(header_part, '/'), + "JWT header contains '/' character"); + RD_UT_ASSERT(!strchr(payload_part, '+'), + "JWT payload contains '+' character"); + RD_UT_ASSERT(!strchr(payload_part, '/'), + "JWT payload contains '/' character"); + RD_UT_ASSERT(!strchr(signature_part, '+'), + "JWT signature contains '+' character"); + RD_UT_ASSERT(!strchr(signature_part, '/'), + "JWT signature contains '/' character"); + + rd_free(header_part); + rd_free(payload_part); + rd_free(signature_part); + rd_free(jwt); + rd_free(rk); + + RD_UT_PASS(); +} + +int unittest_sasl_oauthbearer_oidc_jwt_bearer(void) { + int fails = 0; + + fails += ut_sasl_oauthbearer_oidc_jwt_bearer_base64url_encode(); + fails += ut_sasl_oauthbearer_oidc_jwt_bearer_build_request_body(); + + return fails; +} + +int unittest_sasl_oauthbearer_oidc_assertion(void) { + int fails = 0; + + fails += ut_sasl_oauthbearer_oidc_assertion_parse_from_file(); + fails += ut_sasl_oauthbearer_oidc_assertion_process_template_file(); + fails += ut_sasl_oauthbearer_oidc_assertion_create(); + + return fails; +} diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer_oidc.h b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer_oidc.h similarity index 72% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer_oidc.h rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer_oidc.h index f46bf1beb7a..0127ecb17ef 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_sasl_oauthbearer_oidc.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_sasl_oauthbearer_oidc.h @@ -28,9 +28,24 @@ #ifndef _RDKAFKA_SASL_OAUTHBEARER_OIDC_H_ #define _RDKAFKA_SASL_OAUTHBEARER_OIDC_H_ -void rd_kafka_oidc_token_refresh_cb(rd_kafka_t *rk, - const char *oauthbearer_config, - void *opaque); +void rd_kafka_oidc_token_jwt_bearer_refresh_cb(rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque); + +void rd_kafka_oidc_token_client_credentials_refresh_cb( + rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque); + +void rd_kafka_oidc_token_metadata_azure_imds_refresh_cb( + rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque); + +void rd_kafka_oidc_token_metadata_aws_iam_refresh_cb( + rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque); int unittest_sasl_oauthbearer_oidc(void); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_plain.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl_plain.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_plain.c rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_plain.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_scram.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl_scram.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_scram.c rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_scram.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sasl_win32.c b/lib/librdkafka-2.15.0/src/rdkafka_sasl_win32.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sasl_win32.c rename to lib/librdkafka-2.15.0/src/rdkafka_sasl_win32.c diff --git a/lib/librdkafka-2.15.0/src/rdkafka_share_acknowledgement.c b/lib/librdkafka-2.15.0/src/rdkafka_share_acknowledgement.c new file mode 100644 index 00000000000..35b50d3a57d --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdkafka_share_acknowledgement.c @@ -0,0 +1,1445 @@ +/* + * librdkafka - The Apache Kafka C/C++ library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include "rdkafka_int.h" +#include "rdkafka_metadata.h" +#include "rdkafka_share_acknowledgement.h" + +rd_kafka_share_ack_batch_entry_t * +rd_kafka_share_ack_batch_entry_new(int64_t start_offset, + int64_t end_offset, + int32_t types_cnt, + int16_t delivery_count) { + rd_kafka_share_ack_batch_entry_t *entry; + + entry = rd_calloc(1, sizeof(*entry)); + entry->start_offset = start_offset; + entry->end_offset = end_offset; + entry->size = end_offset - start_offset + 1; + entry->types_cnt = types_cnt; + entry->delivery_count = delivery_count; + entry->types = + rd_calloc((size_t)types_cnt, + sizeof(rd_kafka_share_internal_acknowledgement_type)); + return entry; +} + +void rd_kafka_share_ack_batch_entry_destroy( + rd_kafka_share_ack_batch_entry_t *entry) { + if (!entry) + return; + rd_free(entry->types); + rd_free(entry); +} + +static void rd_kafka_share_ack_batch_entry_destroy_free(void *ptr) { + rd_kafka_share_ack_batch_entry_destroy( + (rd_kafka_share_ack_batch_entry_t *)ptr); +} + +rd_bool_t +rd_kafka_share_acknowledgement_mode_is_implicit(rd_kafka_share_t *rkshare) { + return rkshare->rkshare_rk->rk_conf.share.share_acknowledgement_mode && + !strcmp(rkshare->rkshare_rk->rk_conf.share + .share_acknowledgement_mode, + "implicit"); +} + +rd_bool_t +rd_kafka_share_acknowledgement_mode_is_explicit(rd_kafka_share_t *rkshare) { + return rkshare->rkshare_rk->rk_conf.share.share_acknowledgement_mode && + !strcmp(rkshare->rkshare_rk->rk_conf.share + .share_acknowledgement_mode, + "explicit"); +} + +void rd_kafka_share_acknowledge_all_if_implicit(rd_kafka_share_t *rkshare) { + if (rd_kafka_share_acknowledgement_mode_is_implicit(rkshare)) + rd_kafka_share_ack_all(rkshare); +} + +rd_kafka_error_t * +rd_kafka_share_ensure_all_acknowledged_if_explicit(rd_kafka_share_t *rkshare) { + if (rd_kafka_share_acknowledgement_mode_is_explicit(rkshare) && + rkshare->rkshare_unacked_cnt > 0) + return rd_kafka_error_new( + RD_KAFKA_RESP_ERR__STATE, + "%" PRId64 + " records from previous poll have not " + "been acknowledged", + rkshare->rkshare_unacked_cnt); + return NULL; +} + +rd_kafka_share_ack_batch_entry_t *rd_kafka_share_ack_batch_entry_copy( + const rd_kafka_share_ack_batch_entry_t *src) { + rd_kafka_share_ack_batch_entry_t *dst; + + dst = rd_kafka_share_ack_batch_entry_new( + src->start_offset, src->end_offset, src->types_cnt, + src->delivery_count); + memcpy(dst->types, src->types, + (size_t)src->types_cnt * + sizeof(rd_kafka_share_internal_acknowledgement_type)); + return dst; +} + +static void *rd_kafka_share_ack_batch_entry_copy_void(const void *elem, + void *opaque) { + return rd_kafka_share_ack_batch_entry_copy( + (const rd_kafka_share_ack_batch_entry_t *)elem); +} + +rd_kafka_share_ack_batches_t *rd_kafka_share_ack_batches_new_empty(void) { + return rd_kafka_share_ack_batches_new(NULL, 0, 0); +} + +rd_kafka_share_ack_batches_t * +rd_kafka_share_ack_batches_new(rd_kafka_topic_partition_t *rktpar, + int32_t response_leader_id, + int64_t response_acquired_offsets_count) { + rd_kafka_share_ack_batches_t *batches; + + batches = rd_calloc(1, sizeof(*batches)); + batches->rktpar = rktpar; + batches->response_leader_id = response_leader_id; + batches->response_acquired_offsets_count = + response_acquired_offsets_count; + rd_list_init(&batches->entries, 0, + rd_kafka_share_ack_batch_entry_destroy_free); + return batches; +} + +void rd_kafka_share_ack_batches_destroy(rd_kafka_share_ack_batches_t *batches) { + rd_list_destroy(&batches->entries); + if (batches->rktpar) + rd_kafka_topic_partition_destroy(batches->rktpar); + rd_free(batches); +} + +void rd_kafka_share_ack_batches_destroy_free(void *ptr) { + rd_kafka_share_ack_batches_destroy((rd_kafka_share_ack_batches_t *)ptr); +} + +rd_kafka_share_ack_batches_t * +rd_kafka_share_ack_batches_copy(const rd_kafka_share_ack_batches_t *src) { + rd_kafka_share_ack_batches_t *dst; + + dst = rd_kafka_share_ack_batches_new( + src->rktpar ? rd_kafka_topic_partition_copy(src->rktpar) : NULL, + src->response_leader_id, src->response_acquired_offsets_count); + + /* Deep copy all entries and preserve flags (e.g., RD_LIST_F_SORTED) */ + rd_list_copy_to(&dst->entries, &src->entries, + rd_kafka_share_ack_batch_entry_copy_void, NULL); + dst->entries.rl_flags = src->entries.rl_flags; + return dst; +} + +void *rd_kafka_share_ack_batches_copy_void(const void *elem, void *opaque) { + return rd_kafka_share_ack_batches_copy( + (const rd_kafka_share_ack_batches_t *)elem); +} + +/** + * @brief Transfer inflight acks from response RKO into rkshare's inflight map. + * + * Takes each batch from the response's inflight_acks list and stores it in + * the map (key = topic-partition). Ownership is transferred; the response + * RKO must not free these when destroyed. + * In the future, the map will be cleared per partition after acks are sent. + * + * @param rkshare Share consumer handle + * @param response_rko The share fetch response RKO containing inflight_acks + */ +void rd_kafka_share_build_inflight_acks_map(rd_kafka_share_t *rkshare, + rd_kafka_op_t *response_rko) { + rd_list_t *list = + response_rko->rko_u.share_fetch_response.inflight_acks; + + while (rd_list_cnt(list) > 0) { + rd_kafka_share_ack_batches_t *batches = rd_list_pop(list); + rd_kafka_topic_partition_t *key; + rd_kafka_share_ack_batch_entry_t *entry; + int i, k; + + rd_dassert(batches->rktpar != NULL); + + key = rd_kafka_topic_partition_new_with_id_and_name( + rd_kafka_topic_partition_get_topic_id(batches->rktpar), + batches->rktpar->topic, batches->rktpar->partition); + + /* Each topic-partition is always a new entry (no overwrites). + */ + RD_MAP_SET(&rkshare->rkshare_inflight_acks, key, batches); + + /* Count ACQUIRED types for unacked tracking */ + RD_LIST_FOREACH(entry, &batches->entries, i) { + for (k = 0; k < entry->types_cnt; k++) { + if (entry->types[k] == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED) + rkshare->rkshare_unacked_cnt++; + } + } + } +} + +/** + * @brief Create a new collated batch entry with a single acknowledgement type. + * + * Creates an entry where size=1 and types[0] holds the single ack type + * for the entire offset range [start_offset, end_offset]. + * + * @param start_offset First offset in the range + * @param end_offset Last offset in the range (inclusive) + * @param type The acknowledgement type for the entire range + * + * @returns Newly allocated collated entry (caller must free) + */ +static rd_kafka_share_ack_batch_entry_t * +rd_kafka_share_ack_batch_entry_collated_new( + int64_t start_offset, + int64_t end_offset, + rd_kafka_share_internal_acknowledgement_type type, + int16_t delivery_count) { + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(start_offset, end_offset, 1, + delivery_count); + entry->types[0] = type; + return entry; +} + +/** + * @brief Implicit acknowledgement: convert all ACQUIRED types to ACCEPT. + * + * In implicit ack mode, all records delivered to the application are + * automatically acknowledged as ACCEPT on the next poll(). This function + * walks through all entries in the inflight map and changes ACQUIRED + * types to ACCEPT so that rd_kafka_share_build_ack_details() will + * extract them for sending to the broker. + * + * @param rkshare Share consumer handle + */ +void rd_kafka_share_ack_all(rd_kafka_share_t *rkshare) { + const rd_kafka_topic_partition_t *tp_key; + rd_kafka_share_ack_batches_t *inflight_batches; + + RD_MAP_FOREACH(tp_key, inflight_batches, + &rkshare->rkshare_inflight_acks) { + rd_kafka_share_ack_batch_entry_t *entry; + int i; + rd_kafka_dbg(rkshare->rkshare_rk, CGRP, "SHAREACK", + "Implicit ack: converting ACQUIRED to ACCEPT " + "for %s [%" PRId32 "]", + tp_key->topic, tp_key->partition); + RD_LIST_FOREACH(entry, &inflight_batches->entries, i) { + int k; + for (k = 0; k < entry->types_cnt; k++) { + if (entry->types[k] == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED) + entry->types[k] = + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT; + } + } + } +} + +/** + * @brief Free an ack details batch, its entries, and its owned rktpar. + * + * Used as rd_list_t free callback for ack_details output lists where + * each batch owns a copy of its rktpar. + */ +static void rd_kafka_share_ack_details_batch_destroy(void *ptr) { + rd_kafka_share_ack_batches_t *batch = ptr; + rd_kafka_share_ack_batches_destroy(batch); +} + +/** + * @brief Find an existing ack batch for the same topic-partition in a list. + * + * @param ack_list List of rd_kafka_share_ack_batches_t* + * @param rktpar Topic-partition to match (by topic id and partition) + * + * @returns Matching batch, or NULL if not found. + * @locality main thread + */ +rd_kafka_share_ack_batches_t * +rd_kafka_share_find_ack_batch(rd_list_t *ack_list, + const rd_kafka_topic_partition_t *rktpar) { + rd_kafka_share_ack_batches_t *existing; + int i; + + RD_LIST_FOREACH(existing, ack_list, i) { + if (rd_kafka_topic_partition_by_id_cmp(existing->rktpar, + rktpar) == 0) + return existing; + } + return NULL; +} + +/** + * @brief Find an existing ack batch by topic id and partition. + * + * Same as rd_kafka_share_find_ack_batch but takes topic_id and + * partition directly to avoid allocating a temporary + * rd_kafka_topic_partition_t for the lookup. + * + * TODO KIP-932: Merge this with rd_kafka_share_find_ack_batch and use + * rd_list_find with binary search once ack_list is kept sorted. + * + * @param ack_list List of rd_kafka_share_ack_batches_t*. + * @param topic_id Topic UUID to match. + * @param partition Partition id to match. + * + * @returns Matching batch, or NULL if not found. + */ +rd_kafka_share_ack_batches_t * +rd_kafka_share_find_ack_batch_by_id(rd_list_t *ack_list, + rd_kafka_Uuid_t topic_id, + int32_t partition) { + rd_kafka_share_ack_batches_t *existing; + int i; + + RD_LIST_FOREACH(existing, ack_list, i) { + if (existing->rktpar->partition == partition && + !rd_kafka_Uuid_cmp( + rd_kafka_topic_partition_get_topic_id(existing->rktpar), + topic_id)) + return existing; + } + return NULL; +} + +/** + * @brief Reply to a SHARE_FETCH op with an error, propagating the + * error to each batch in ack_details. + * + * On top-level error (err != 0): + * - If err is BAD_MSG / __UNDERFLOW (wire-level parse failure): + * override every batch with INVALID_RECORD_STATE so the consumer + * treats these as "outcome unknown" and re-acquires; any + * per-partition err the parser may have written before failing + * could be a misaligned read and is no longer trustworthy. + * - Otherwise: propagate err to each batch still at _IN_PROGRESS, + * preserving any deliberately pre-set err and any per-partition + * err already written by the parser. + * + * On success (err == 0), the per-partition errors have already been + * set by the response parser and we leave ack_details untouched. + */ +void rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rd_kafka_op_t *rko, + rd_kafka_resp_err_t err) { + if (err && rko->rko_u.share_fetch.ack_details) { + rd_kafka_share_ack_batches_t *batch; + int i; + + if (likely(err != RD_KAFKA_RESP_ERR__BAD_MSG && + err != RD_KAFKA_RESP_ERR__UNDERFLOW)) { + + /* Propagate top-level err to each batch in + * ack_details, but only override the _IN_PROGRESS + * sentinel. This preserves: + * - Deliberately pre-set err (e.g. + * INVALID_SHARE_SESSION_EPOCH from the + * epoch-0 strip path). + * - Per-partition err already written by the + * parser. */ + RD_LIST_FOREACH(batch, + rko->rko_u.share_fetch.ack_details, i) { + if (batch->rktpar->err == + RD_KAFKA_RESP_ERR__IN_PROGRESS) + batch->rktpar->err = err; + } + } else { + /* Wire-level parse failure: response framing is + * untrusted, so any per-partition err the parser + * may have written before failing could be a + * misaligned read. */ + RD_LIST_FOREACH(batch, + rko->rko_u.share_fetch.ack_details, i) { + batch->rktpar->err = err; + } + } + } + rd_kafka_op_reply(rko, err); +} + +/** + * @brief Whether the leader cached for the partition has diverged + * from the leader at which \p batch's records were acquired. + */ +static rd_bool_t +rd_kafka_share_ack_batch_leader_stale(rd_kafka_share_ack_batches_t *batch, + int32_t current_leader_id) { + return current_leader_id != batch->response_leader_id; +} + +/** + * @brief Resolve the leader broker for an ack batch, or fail the + * batch locally with the appropriate error code. + * + * Snapshots (rktp_leader, rktp_leader_id) under rktp_lock to + * avoid racing with concurrent leader-state clearing on the + * broker thread. On success, returns a kept broker handle; + * the caller MUST rd_kafka_broker_destroy() it after use. + * On failure returns NULL with \p *errp set. + * + * @locality main thread + */ +static rd_kafka_broker_t *rd_kafka_share_ack_batch_resolve_leader_or_fail_acks( + rd_kafka_t *rk, + rd_kafka_share_ack_batches_t *batch, + rd_kafka_resp_err_t *errp) { + rd_kafka_toppar_t *rktp; + rd_kafka_broker_t *leader_rkb; + int32_t leader_id; + + rktp = rd_kafka_topic_partition_toppar(rk, batch->rktpar); + + /* Defensive: in production the rktp is always attached to the + * batch by the ShareFetch reply parser via toppar_keep, so + * this branch should never fire. Trigger a cluster-wide + * metadata refresh to recover if it ever does. */ + if (unlikely(!rktp)) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Ack batch for partition %" PRId32 + " dropped: toppar not found " + "(should be unreachable)", + batch->rktpar->partition); + rd_kafka_metadata_refresh_known_topics( + rk, NULL, rd_false /*don't force*/, + "shareack toppar not found"); + *errp = RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART; + return NULL; + } + + /* Snapshot (rktp_leader, rktp_leader_id) under rktp_lock to + * avoid racing with concurrent leader-state clearing on the + * broker thread. */ + rd_kafka_toppar_lock(rktp); + leader_rkb = rktp->rktp_leader; + leader_id = rktp->rktp_leader_id; + if (leader_rkb) + rd_kafka_broker_keep(leader_rkb); + rd_kafka_toppar_unlock(rktp); + + /* Cached leader differs from the leader at which records + * were acquired. The session for these records lives on the + * original broker which no longer leads this partition. No + * metadata refresh - the cached current leader is already + * the new one (that's how this check fired). */ + if (unlikely(rd_kafka_share_ack_batch_leader_stale(batch, leader_id))) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Ack batch for %s [%" PRId32 + "] dropped locally: leader changed " + "since records were acquired " + "(was %" PRId32 ", now %" PRId32 ")", + batch->rktpar->topic, batch->rktpar->partition, + batch->response_leader_id, leader_id); + if (leader_rkb) + rd_kafka_broker_destroy(leader_rkb); + *errp = RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER; + return NULL; + } + + /* Defensive: no cached leader. Unreachable in practice + * because leader-stale fires first when the cached leader id + * is -1, but kept to make the intent explicit and survive + * future invariant changes. Trigger a metadata refresh and + * surface NOT_LEADER_OR_FOLLOWER. */ + if (unlikely(!leader_rkb)) { + rd_kafka_dbg(rk, CGRP, "SHARE", + "Ack batch for %s [%" PRId32 + "] dropped: no cached leader, " + "triggering metadata refresh", + batch->rktpar->topic, batch->rktpar->partition); + rd_kafka_toppar_leader_unavailable( + rktp, "shareack", RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER); + *errp = RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER; + return NULL; + } + + /* Cached leader broker is being decommissioned or the + * instance is terminating. Surface NOT_LEADER_OR_FOLLOWER and + * trigger a metadata refresh so the next request finds the + * new leader. */ + if (unlikely(rd_kafka_broker_or_instance_terminating(leader_rkb))) { + rd_kafka_dbg( + rk, CGRP, "SHARE", + "Ack batch for %s [%" PRId32 "] dropped: broker %" PRId32 + " is terminating, " + "triggering metadata refresh", + batch->rktpar->topic, batch->rktpar->partition, leader_id); + rd_kafka_toppar_leader_unavailable( + rktp, "shareack", RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER); + rd_kafka_broker_destroy(leader_rkb); + *errp = RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER; + return NULL; + } + + *errp = RD_KAFKA_RESP_ERR_NO_ERROR; + return leader_rkb; +} + + +/** + * @brief Segregate sync ack batches by partition leader into + * each broker's pending_commit_sync list. + * + * Phase 1 of sync dispatch: puts ALL acks into rkb_pending_commit_sync + * regardless of whether the broker is free or busy. Updates the + * commit_sync results for partitions with no available leader. + * + * @param rk Client instance. + * @param rkcg Consumer group handle. + * @param ack_batches List of ack batches from the SYNC_FANOUT op. + * Elements are moved to broker lists; container destroyed. + * @param abs_timeout Absolute timeout for the commit_sync request. + * + * @locality main thread + */ +void rd_kafka_share_segregate_sync_acks_by_leader(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg, + rd_list_t *ack_batches, + rd_ts_t abs_timeout) { + rd_kafka_share_ack_batches_t *batch; + int batch_cnt = rd_list_cnt(ack_batches); + + while ((batch = rd_list_pop(ack_batches))) { + rd_kafka_broker_t *leader_rkb; + rd_kafka_share_ack_batches_t *existing; + rd_kafka_topic_partition_t *result_rktpar; + rd_kafka_resp_err_t fail_err; + + leader_rkb = + rd_kafka_share_ack_batch_resolve_leader_or_fail_acks( + rk, batch, &fail_err); + if (!leader_rkb) { + result_rktpar = rd_kafka_topic_partition_list_find( + rkcg->rkcg_commit_sync_request.results, + batch->rktpar->topic, batch->rktpar->partition); + if (result_rktpar) + result_rktpar->err = fail_err; + else + rd_kafka_dbg(rk, CGRP, "SHARE", + "Sync ack batch for %s [%" PRId32 + "]: partition not found in " + "commit_sync results", + batch->rktpar->topic, + batch->rktpar->partition); + + rd_kafka_share_enqueue_ack_commit_cb_op(rk, batch, + fail_err); + rd_kafka_share_ack_batches_destroy(batch); + continue; + } + + /* Put all acks into pending_commit_sync */ + if (!leader_rkb->rkb_pending_commit_sync.sync_ack_details) + leader_rkb->rkb_pending_commit_sync.sync_ack_details = + rd_list_new( + batch_cnt, + rd_kafka_share_ack_batches_destroy_free); + + existing = rd_kafka_share_find_ack_batch( + leader_rkb->rkb_pending_commit_sync.sync_ack_details, + batch->rktpar); + if (existing) { + rd_kafka_share_ack_batch_entry_t *entry; + int j; + RD_LIST_FOREACH(entry, &batch->entries, j) { + rd_list_add( + &existing->entries, + rd_kafka_share_ack_batch_entry_copy(entry)); + } + rd_list_sort(&existing->entries, + rd_kafka_share_ack_entries_sort_cmp_ptr); + rd_kafka_share_ack_batches_destroy(batch); + } else { + rd_list_add(leader_rkb->rkb_pending_commit_sync + .sync_ack_details, + batch); + } + + leader_rkb->rkb_pending_commit_sync.abs_timeout = abs_timeout; + leader_rkb->rkb_pending_commit_sync.commit_sync_request_id = + rkcg->rkcg_commit_sync_request.id; + + rd_kafka_broker_destroy(leader_rkb); + } + + rd_list_destroy(ack_batches); +} + + +/** + * @brief Segregate ack batches from a FANOUT op by partition leader. + * + * For each ack batch, looks up the current leader broker via the + * toppar reference in batch->rktpar, and merges the batch into that + * broker's rkb_share_async_ack_details list. If the broker already + * has cached acks for the same topic-partition, the entries are + * appended to the existing batch. + * + * @param rk Client instance + * @param ack_batches List of rd_kafka_share_ack_batches_t* from FANOUT op. + * Elements whose leader is found are moved to broker + * ack_details; remaining elements are not freed. + * + * @locality main thread + */ +void rd_kafka_share_segregate_acks_by_leader(rd_kafka_t *rk, + rd_list_t *ack_batches) { + rd_kafka_share_ack_batches_t *batch; + int batch_cnt = rd_list_cnt(ack_batches); + + while ((batch = rd_list_pop(ack_batches))) { + rd_kafka_broker_t *leader_rkb; + rd_kafka_share_ack_batches_t *existing; + rd_kafka_resp_err_t fail_err; + + leader_rkb = + rd_kafka_share_ack_batch_resolve_leader_or_fail_acks( + rk, batch, &fail_err); + if (!leader_rkb) { + rd_kafka_share_enqueue_ack_commit_cb_op(rk, batch, + fail_err); + rd_kafka_share_ack_batches_destroy(batch); + continue; + } + + /* Allocate list on first use with incoming batch count + * as initial capacity hint */ + if (!leader_rkb->rkb_share_async_ack_details) + leader_rkb->rkb_share_async_ack_details = rd_list_new( + batch_cnt, rd_kafka_share_ack_batches_destroy_free); + + /* Check if there's already a batch for this topic-partition. + * If so, merge entries; otherwise add as new. */ + existing = rd_kafka_share_find_ack_batch( + leader_rkb->rkb_share_async_ack_details, batch->rktpar); + + if (existing) { + /* Merge: deep-copy entries from new batch into + * existing, preserving order. The source batch + * is then fully destroyed (freeing its entries). */ + rd_kafka_share_ack_batch_entry_t *entry; + int j; + RD_LIST_FOREACH(entry, &batch->entries, j) { + rd_list_add( + &existing->entries, + rd_kafka_share_ack_batch_entry_copy(entry)); + } + rd_kafka_share_ack_batches_destroy(batch); + } else { + rd_list_add(leader_rkb->rkb_share_async_ack_details, + batch); + } + + rd_kafka_broker_destroy(leader_rkb); + } +} + +/** + * @brief Extract acknowledged (non-ACQUIRED) records from inflight map. + * + * Iterates through the inflight acknowledgement map and separates each + * entry's per-offset types into: + * - Non-ACQUIRED offsets: collated into ack_details for sending to broker + * (consecutive same-type offsets merged into single entry with 1 type) + * - ACQUIRED offsets: kept in the map as new entries (per-offset types) + * + * Map entries with no remaining ACQUIRED offsets are removed. + * If the map becomes empty after processing, it is cleared. + * + * @param rkshare Share consumer handle + * @returns Allocated list of rd_kafka_share_ack_batches_t*, or NULL if + * there are no ack details to send. Caller must destroy. + * TODO KIP-932: Change name. + */ +rd_list_t *rd_kafka_share_build_ack_details(rd_kafka_share_t *rkshare) { + const rd_kafka_topic_partition_t *tp_key; + rd_kafka_share_ack_batches_t *inflight_batches; + rd_list_t keys_to_delete; + rd_list_t *ack_details = NULL; + int i; + rd_kafka_topic_partition_t *del_key; + + rd_list_init(&keys_to_delete, 0, NULL); + + rkshare->rkshare_unacked_cnt = 0; + + RD_MAP_FOREACH(tp_key, inflight_batches, + &rkshare->rkshare_inflight_acks) { + rd_kafka_share_ack_batches_t *ack_batch = NULL; + rd_kafka_share_ack_batch_entry_t *entry; + rd_list_t new_entries; + int ei; + + rd_kafka_dbg(rkshare->rkshare_rk, CGRP, "SHAREACK", + "Building ack details for %s [%" PRId32 "]", + tp_key->topic, tp_key->partition); + + rd_list_init(&new_entries, 0, + rd_kafka_share_ack_batch_entry_destroy_free); + + RD_LIST_FOREACH(entry, &inflight_batches->entries, ei) { + int64_t j = 0; + + while (j < entry->types_cnt) { + rd_kafka_share_internal_acknowledgement_type + run_type = entry->types[j]; + int64_t run_start = entry->start_offset + j; + int64_t k = j + 1; + + /* Find end of consecutive same-type run */ + while (k < entry->types_cnt && + entry->types[k] == run_type) + k++; + + int64_t run_end = entry->start_offset + k - 1; + + if (run_type == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED) { + /* ACQUIRED: keep in map */ + int32_t cnt = (int32_t)(k - j); + rd_kafka_share_ack_batch_entry_t + *new_entry = + rd_kafka_share_ack_batch_entry_new( + run_start, run_end, cnt, + entry->delivery_count); + int32_t m; + for (m = 0; m < cnt; m++) + new_entry->types[m] = + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + rd_list_add(&new_entries, new_entry); + rkshare->rkshare_unacked_cnt += cnt; + } else { + /* Non-ACQUIRED: add to ack_details + */ + if (!ack_batch) { + ack_batch = + rd_kafka_share_ack_batches_new( + rd_kafka_topic_partition_copy( + inflight_batches + ->rktpar), + inflight_batches + ->response_leader_id, + 0); + /* Sentinel: no reply path + * has touched this batch + * yet. Distinct from + * INVALID_RECORD_STATE + * (set in buf callback) + * which means "reply + * received but partition + * missing from response". + * Defensive check in main + * reply handler converts + * any remaining + * _IN_PROGRESS to the + * top-level rko_err so + * silent-destroy paths + * (q_enq disabled queue, + * helper bypassed) don't + * leak the sentinel to + * the app. */ + ack_batch->rktpar->err = + RD_KAFKA_RESP_ERR__IN_PROGRESS; + } + rd_kafka_dbg( + rkshare->rkshare_rk, CGRP, + "SHAREACK", + " Adding ack detail for offsets " + "[%" PRId64 " - %" PRId64 + "] with type %d", + run_start, run_end, run_type); + /* Collated: 1 type for entire + * range */ + rd_list_add( + &ack_batch->entries, + rd_kafka_share_ack_batch_entry_collated_new( + run_start, run_end, run_type, + entry->delivery_count)); + } + + j = k; + } + } + + /** + * TODO KIP-932: Check if it can be optimized as these are + * internal thread operations and they should + * be as fast as possible as this will be called + * again and again. + */ + if (rd_list_is_sorted(&new_entries, + rd_kafka_share_ack_entries_sort_cmp_ptr)) + new_entries.rl_flags |= RD_LIST_F_SORTED; + + /* Replace inflight entries with ACQUIRED-only entries */ + rd_list_destroy(&inflight_batches->entries); + inflight_batches->entries = new_entries; + + /* If no ACQUIRED offsets remain, mark for removal */ + if (rd_list_cnt(&inflight_batches->entries) == 0) { + rd_kafka_topic_partition_t *del_key = + rd_kafka_topic_partition_new_with_id_and_name( + rd_kafka_topic_partition_get_topic_id( + inflight_batches->rktpar), + inflight_batches->rktpar->topic, + inflight_batches->rktpar->partition); + rd_list_add(&keys_to_delete, del_key); + } + + if (ack_batch) { + if (!ack_details) + ack_details = rd_list_new( + 0, + rd_kafka_share_ack_details_batch_destroy); + rd_list_add(ack_details, ack_batch); + } + } + + /* Remove map entries with no remaining ACQUIRED offsets */ + RD_LIST_FOREACH(del_key, &keys_to_delete, i) { + RD_MAP_DELETE(&rkshare->rkshare_inflight_acks, del_key); + rd_kafka_topic_partition_destroy(del_key); + } + rd_list_destroy(&keys_to_delete); + + /* Clear map if empty */ + if (RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 0) + RD_MAP_CLEAR(&rkshare->rkshare_inflight_acks); + + return ack_details; +} + +/** + * @brief Update acknowledgement type for a specific offset within an entry. + * + * Handles the transition from ACQUIRED state by decrementing the unacked count. + * + * @param rkshare Share consumer handle. + * @param entry The batch entry containing the offset. + * @param idx Index within the entry's types array. + * @param type New acknowledgement type. + */ +static void rd_kafka_share_update_acknowledgement_type( + rd_kafka_share_t *rkshare, + rd_kafka_share_ack_batch_entry_t *entry, + int64_t idx, + rd_kafka_share_AcknowledgeType_t type) { + /* Decrement unacked count when transitioning from ACQUIRED */ + if (entry->types[idx] == RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED) + rkshare->rkshare_unacked_cnt--; + + /* Update the type */ + entry->types[idx] = (rd_kafka_share_internal_acknowledgement_type)type; +} + +/** + * @brief Comparator for sorting/checking entries by start_offset. + * + * Used with rd_list_sort() and rd_list_is_sorted(). + */ +int rd_kafka_share_ack_entries_sort_cmp_ptr(const void *_a, const void *_b) { + const rd_kafka_share_ack_batch_entry_t *a = + (const rd_kafka_share_ack_batch_entry_t *)_a; + const rd_kafka_share_ack_batch_entry_t *b = + (const rd_kafka_share_ack_batch_entry_t *)_b; + + if (a->start_offset < b->start_offset) + return -1; + if (a->start_offset > b->start_offset) + return 1; + return 0; +} + +/** + * @brief Comparator for finding an entry containing a given offset. + * + * Used with rd_list_find() for binary search when RD_LIST_F_SORTED is set. + * + * @param _offset Pointer to the offset being searched (int64_t *) + * @param _entry Pointer to the batch entry (rd_kafka_share_ack_batch_entry_t *) + * + * @returns negative if offset < entry->start_offset + * @returns positive if offset > entry->end_offset + * @returns 0 if offset is within [start_offset, end_offset] + */ +static int rd_kafka_share_ack_entries_offset_find_cmp_ptr(const void *_offset, + const void *_entry) { + const int64_t *offset = (const int64_t *)_offset; + const rd_kafka_share_ack_batch_entry_t *entry = + (const rd_kafka_share_ack_batch_entry_t *)_entry; + + if (*offset < entry->start_offset) + return -1; + if (*offset > entry->end_offset) + return 1; + return 0; +} + +/** + * @brief Look up a batch entry containing the given offset. + * + * Finds the partition in the inflight_acks map and locates the entry + * containing the specified offset using binary search. + * + * @param rkshare Share consumer handle. + * @param topic Topic name. + * @param partition Partition id. + * @param offset Offset to find. + * @param entry_out Output parameter for the found entry. + * @param idx_out Output parameter for the index within the entry. + * + * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, + * RD_KAFKA_RESP_ERR__STATE if partition or offset not found. + */ +static rd_kafka_resp_err_t +rd_kafka_share_find_ack_entry(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t offset, + rd_kafka_share_ack_batch_entry_t **entry_out, + int64_t *idx_out) { + rd_kafka_topic_partition_t *lookup_key; + rd_kafka_share_ack_batches_t *batches; + rd_kafka_share_ack_batch_entry_t *entry; + + /* Find partition in inflight_acks map */ + lookup_key = rd_kafka_topic_partition_new(topic, partition); + batches = RD_MAP_GET(&rkshare->rkshare_inflight_acks, lookup_key); + rd_kafka_topic_partition_destroy(lookup_key); + if (!batches) + return RD_KAFKA_RESP_ERR__STATE; + + /* Find entry containing offset using binary search. + * Entries are sorted by start_offset and don't overlap. */ + entry = rd_list_find(&batches->entries, &offset, + rd_kafka_share_ack_entries_offset_find_cmp_ptr); + if (!entry) + return RD_KAFKA_RESP_ERR__STATE; + + *entry_out = entry; + *idx_out = offset - entry->start_offset; + + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +/** + * @brief Internal acknowledge-offset helper. + * + * Performs argument validation, inflight-map lookup, and type update. + * Does NOT take the share consumer access gate — callers must call + * rd_kafka_share_acquire() / rd_kafka_share_release() around this. + */ +static rd_kafka_resp_err_t +rd_kafka_share_acknowledge_offset0(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t offset, + rd_kafka_share_AcknowledgeType_t type) { + rd_kafka_share_ack_batch_entry_t *entry; + int64_t idx; + rd_kafka_resp_err_t err; + + if (unlikely((err = rd_kafka_share_consumer_closed_err(rkshare)))) + return err; + + if (!topic || partition < 0 || offset < 0) + return RD_KAFKA_RESP_ERR__INVALID_ARG; + + /* Explicit acknowledge APIs require explicit acknowledgement mode */ + if (rd_kafka_share_acknowledgement_mode_is_implicit(rkshare)) + return RD_KAFKA_RESP_ERR__STATE; + + /* Validate type - ACCEPT, RELEASE, REJECT allowed */ + if (type < RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT || + type > RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT) + return RD_KAFKA_RESP_ERR__INVALID_ARG; + + /* Find partition and entry containing the offset */ + err = rd_kafka_share_find_ack_entry(rkshare, topic, partition, offset, + &entry, &idx); + if (err) + return err; + + /* GAP records cannot be acknowledged */ + if (entry->types[idx] == RD_KAFKA_SHARE_INTERNAL_ACK_GAP) + return RD_KAFKA_RESP_ERR__STATE; + + rd_kafka_share_update_acknowledgement_type(rkshare, entry, idx, type); + + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +rd_kafka_resp_err_t +rd_kafka_share_acknowledge(rd_kafka_share_t *rkshare, + const rd_kafka_message_t *rkmessage) { + return rd_kafka_share_acknowledge_type( + rkshare, rkmessage, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); +} + +rd_kafka_resp_err_t +rd_kafka_share_acknowledge_type(rd_kafka_share_t *rkshare, + const rd_kafka_message_t *rkmessage, + rd_kafka_share_AcknowledgeType_t type) { + rd_kafka_resp_err_t err; + rd_kafka_error_t *error = NULL; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) { + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + return err; + } + + if (!rkmessage) { + err = RD_KAFKA_RESP_ERR__INVALID_ARG; + goto done; + } + + if (!rkmessage->rkt) { + err = RD_KAFKA_RESP_ERR__STATE; + goto done; + } + + err = rd_kafka_share_acknowledge_offset0( + rkshare, rd_kafka_topic_name(rkmessage->rkt), rkmessage->partition, + rkmessage->offset, type); + +done: + rd_kafka_share_release(rkshare); + return err; +} + +rd_kafka_resp_err_t +rd_kafka_share_acknowledge_offset(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t offset, + rd_kafka_share_AcknowledgeType_t type) { + rd_kafka_resp_err_t err; + rd_kafka_error_t *error = NULL; + + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) { + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + return err; + } + + err = rd_kafka_share_acknowledge_offset0(rkshare, topic, partition, + offset, type); + + rd_kafka_share_release(rkshare); + return err; +} + + +/** + * @brief Initialize a partition offsets element with topicid,topic, partition, + * and offsets. + * + * @param topic_id Topic UUID. + * @param topic Topic name (will be duplicated). + * @param partition Partition id. + * @param offsets_cnt Number of offsets to allocate space for. + */ +static rd_kafka_share_partition_offsets_t * +rd_kafka_share_partition_offsets_new(rd_kafka_topic_partition_t *tp, + size_t offsets_cnt) { + rd_kafka_share_partition_offsets_t *elem; + size_t size; + + /* Allocate struct + flexible array in one allocation */ + size = sizeof(*elem) + (offsets_cnt * sizeof(int64_t)); + elem = rd_calloc(1, size); + + elem->partition = rd_kafka_topic_partition_new_with_id_and_name( + rd_kafka_topic_partition_get_topic_id(tp), tp->topic, + tp->partition); + elem->cnt = offsets_cnt; + + return elem; +} + +/** + * @brief Destroy a partition offsets element. + * + * @param elem Element to destroy. + */ +static void rd_kafka_share_partition_offsets_destroy( + rd_kafka_share_partition_offsets_t *elem) { + if (!elem) + return; + rd_kafka_topic_partition_destroy(elem->partition); + rd_free(elem); +} + +/** + * @brief Allocate and initialize a partition offsets list with given capacity. + * + * @param capacity Number of elements to allocate space for. + * @returns Newly allocated list, or NULL if capacity is 0. + * Caller must destroy with + * rd_kafka_share_partition_offsets_list_destroy(). + */ +static rd_kafka_share_partition_offsets_list_t * +rd_kafka_share_partition_offsets_list_new(size_t capacity) { + rd_kafka_share_partition_offsets_list_t *list; + size_t size; + + /* Allocate list + flexible array in one allocation */ + size = sizeof(*list) + + (capacity * sizeof(rd_kafka_share_partition_offsets_t *)); + list = rd_calloc(1, size); + + return list; +} + +rd_kafka_share_partition_offsets_list_t * +rd_kafka_share_build_partition_offsets_list( + rd_kafka_share_ack_batches_t *batches) { + rd_kafka_share_partition_offsets_list_t *list; + rd_kafka_share_partition_offsets_t *elem; + rd_kafka_share_ack_batch_entry_t *entry; + int total_offsets = 0; + int offset_idx = 0; + int j; + + /* Count total offsets */ + RD_LIST_FOREACH(entry, &batches->entries, j) { + total_offsets += + (int)(entry->end_offset - entry->start_offset + 1); + } + + list = rd_kafka_share_partition_offsets_list_new(1); + + /* Allocate elements */ + elem = rd_kafka_share_partition_offsets_new(batches->rktpar, + total_offsets); + + list->cnt = 1; + list->elems[0] = elem; + + /* Fill offsets array */ + RD_LIST_FOREACH(entry, &batches->entries, j) { + int64_t off; + for (off = entry->start_offset; off <= entry->end_offset; + off++) { + elem->offsets[offset_idx++] = off; + } + } + + return list; +} + + +void rd_kafka_share_enqueue_ack_commit_cb_op( + rd_kafka_t *rk, + rd_kafka_share_ack_batches_t *batches, + rd_kafka_resp_err_t err) { + rd_kafka_op_t *cb_rko; + rd_kafka_share_partition_offsets_list_t *partitions; + + /* Check if a runtime callback is registered. + * Locality: main thread - reads flag owned by main thread. + * No race because the flag is only modified by the main thread + * via RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_REGISTER op handler. */ + if (!rk->rk_rkshare || + !rk->rk_share_consumer.acknowledgement_commit_cb_registered) + return; + + partitions = rd_kafka_share_build_partition_offsets_list(batches); + + cb_rko = rd_kafka_op_new(RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE); + cb_rko->rko_err = err; + cb_rko->rko_u.share_ack_commit_cb_execute.partitions = partitions; + + rd_kafka_q_enq(rk->rk_rep, cb_rko); +} + + +size_t rd_kafka_share_partition_offsets_list_count( + const rd_kafka_share_partition_offsets_list_t *list) { + return list->cnt; +} + + +const rd_kafka_share_partition_offsets_t * +rd_kafka_share_partition_offsets_list_get( + const rd_kafka_share_partition_offsets_list_t *list, + size_t index) { + return list->elems[index]; +} + + +void rd_kafka_share_partition_offsets_list_destroy( + rd_kafka_share_partition_offsets_list_t *list) { + size_t i; + + if (!list) + return; + + /* Free each element */ + for (i = 0; i < list->cnt; i++) { + rd_kafka_share_partition_offsets_destroy(list->elems[i]); + } + /* Free the list (elems array is inline via FAM) */ + rd_free(list); +} + + +const rd_kafka_topic_partition_t *rd_kafka_share_partition_offsets_partition( + const rd_kafka_share_partition_offsets_t *partition_offsets) { + return partition_offsets->partition; +} + + +const int64_t *rd_kafka_share_partition_offsets_offsets( + const rd_kafka_share_partition_offsets_t *partition_offsets) { + return partition_offsets->offsets; +} + + +size_t rd_kafka_share_partition_offsets_offsets_cnt( + const rd_kafka_share_partition_offsets_t *partition_offsets) { + return partition_offsets->cnt; +} + + +/** + * @brief Translate librdkafka-internal err sentinels into the broker-equivalent + * codes the application is expected to handle. + */ +static rd_kafka_resp_err_t +rd_kafka_share_translate_app_err(rd_kafka_resp_err_t err) { + switch (err) { + case RD_KAFKA_RESP_ERR__TIMED_OUT: + case RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE: + return RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + case RD_KAFKA_RESP_ERR__DESTROY: + case RD_KAFKA_RESP_ERR__DESTROY_BROKER: + return RD_KAFKA_RESP_ERR__TRANSPORT; + case RD_KAFKA_RESP_ERR__BAD_MSG: + return RD_KAFKA_RESP_ERR_INVALID_MSG; + case RD_KAFKA_RESP_ERR__AUTHENTICATION: + return RD_KAFKA_RESP_ERR_SASL_AUTHENTICATION_FAILED; + default: + return err; + } +} + +void rd_kafka_share_dispatch_ack_callbacks(rd_kafka_t *rk, + rd_list_t *ack_details) { + rd_kafka_share_ack_batches_t *ack_batch; + int k; + + /* Locality: main thread - checks runtime-set registration flag. */ + /** + * TODO KIP-932: Check if we want to send individual ops or 1 op. + */ + if (!rk->rk_share_consumer.acknowledgement_commit_cb_registered || + !ack_details || rd_list_cnt(ack_details) == 0) + return; + + /* Use per-partition error from each batch, translating internal + * sentinels to their broker-equivalent codes before the app sees + * them. */ + RD_LIST_FOREACH(ack_batch, ack_details, k) { + ack_batch->rktpar->err = + rd_kafka_share_translate_app_err(ack_batch->rktpar->err); + rd_kafka_share_enqueue_ack_commit_cb_op(rk, ack_batch, + ack_batch->rktpar->err); + } +} + +/** + * @brief Send commit_sync response to the app thread and clear state. + * + * Creates a SHARE_COMMIT_SYNC_FANOUT reply op with the per-partition + * results and enqueues it on the app thread's temp queue. Clears the + * commit_sync request state. + * + * @param rkcg Consumer group handle. + * + * @locality main thread + */ +void rd_kafka_share_commit_sync_send_response(rd_kafka_cgrp_t *rkcg) { + rd_kafka_op_t *rko_reply; + rd_kafka_topic_partition_list_t *results = + rkcg->rkcg_commit_sync_request.results; + int i; + + /* Translate internal sentinels to broker-equivalent codes before + * the app sees them. This is the unique fan-in for every + * commit_sync result: every writer (apply_result, segregate-fail, + * api-timer-cb, broker decommission) lands its err in this list + * before we ship it back to the caller. */ + for (i = 0; i < results->cnt; i++) + results->elems[i].err = + rd_kafka_share_translate_app_err(results->elems[i].err); + + rko_reply = rd_kafka_op_new(RD_KAFKA_OP_SHARE_COMMIT_SYNC_FANOUT_REPLY); + rko_reply->rko_u.share_commit_sync_fanout_reply.results = results; + + rd_kafka_q_enq(rkcg->rkcg_commit_sync_request.replyq, rko_reply); + rd_kafka_q_destroy(rkcg->rkcg_commit_sync_request.replyq); + + rkcg->rkcg_commit_sync_request.id = 0; + rkcg->rkcg_commit_sync_request.results = NULL; + rkcg->rkcg_commit_sync_request.replyq = NULL; + rkcg->rkcg_commit_sync_request.abs_timeout = 0; + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt = 0; +} + +/** + * @brief Check if all broker results are in and send response if done. + * + * Called after each broker reply arrives. If brokers_awaiting_result_cnt + * reaches zero, stops the timeout timer and sends the response op + * to the app thread's temp queue. + * + * @param rk Client instance. + * @param rkcg Consumer group handle. + * + * @locality main thread + */ +void rd_kafka_share_commit_sync_maybe_complete(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg) { + if (rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt > 0) + return; + + rd_kafka_dbg(rk, CGRP, "SHARE", + "Commit sync request %" PRId64 + " complete; all broker replies received, " + "sending response", + rkcg->rkcg_commit_sync_request.id); + + rd_kafka_timer_stop(&rk->rk_timers, &rkcg->rkcg_commit_sync_request.tmr, + 1); + + rd_kafka_share_commit_sync_send_response(rkcg); +} + +/** + * @brief Apply a broker's commit_sync result: copy each batch's + * per-partition err onto the corresponding entry in + * rkcg_commit_sync_request.results, decrement the count of + * brokers still awaiting reply, and complete the commit_sync + * if this was the last broker outstanding. + * + * The caller is responsible for setting batch->rktpar->err on each + * batch before invoking this (either from the broker reply path or + * from a synthetic failure path like broker decommission). + * + * @locality main thread. + */ +void rd_kafka_share_commit_sync_apply_result(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg, + rd_list_t *ack_batches) { + if (ack_batches) { + rd_kafka_share_ack_batches_t *batch; + int k; + + RD_LIST_FOREACH(batch, ack_batches, k) { + rd_kafka_topic_partition_t *dst = + rd_kafka_topic_partition_list_find( + rkcg->rkcg_commit_sync_request.results, + batch->rktpar->topic, batch->rktpar->partition); + if (dst) + dst->err = batch->rktpar->err; + } + } + + rkcg->rkcg_commit_sync_request.brokers_awaiting_result_cnt--; + rd_kafka_share_commit_sync_maybe_complete(rk, rkcg); +} + + +void rd_kafka_share_acks_clear_during_broker_decommission( + rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_kafka_share_ack_batches_t *batch; + int k; + + /* Async acks: stamp SHARE_SESSION_NOT_FOUND on each batch and + * fire the share-ack callback so the application sees the + * failure. */ + if (rkb->rkb_share_async_ack_details) { + rd_rkb_dbg(rkb, BROKER, "TERM", + "Clearing %d pending async ack batch(es); " + "dispatching ack callbacks with " + "SHARE_SESSION_NOT_FOUND", + rd_list_cnt(rkb->rkb_share_async_ack_details)); + + RD_LIST_FOREACH(batch, rkb->rkb_share_async_ack_details, k) { + batch->rktpar->err = + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND; + } + + rd_kafka_share_dispatch_ack_callbacks( + rk, rkb->rkb_share_async_ack_details); + + rd_list_destroy(rkb->rkb_share_async_ack_details); + rkb->rkb_share_async_ack_details = NULL; + } + + /* Sync acks: stamp SHARE_SESSION_NOT_FOUND on each batch and + * apply the result to the in-flight commit_sync request (copies + * err into the cgrp's result list, decrements awaiting count, + * completes the sync if this was the last broker outstanding). */ + if (rkb->rkb_pending_commit_sync.sync_ack_details) { + rd_kafka_cgrp_t *rkcg = rd_kafka_cgrp_get(rk); + + rd_rkb_dbg( + rkb, BROKER, "TERM", + "Clearing %d pending commit sync ack batch(es); " + "dispatching ack callbacks and " + "stamping SHARE_SESSION_NOT_FOUND on commit_sync result " + "for this broker's partitions", + rd_list_cnt(rkb->rkb_pending_commit_sync.sync_ack_details)); + + RD_LIST_FOREACH( + batch, rkb->rkb_pending_commit_sync.sync_ack_details, k) { + batch->rktpar->err = + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND; + } + + rd_kafka_share_commit_sync_apply_result( + rk, rkcg, rkb->rkb_pending_commit_sync.sync_ack_details); + + rd_kafka_share_dispatch_ack_callbacks( + rk, rkb->rkb_pending_commit_sync.sync_ack_details); + + rd_list_destroy(rkb->rkb_pending_commit_sync.sync_ack_details); + rkb->rkb_pending_commit_sync.sync_ack_details = NULL; + } +} diff --git a/lib/librdkafka-2.15.0/src/rdkafka_share_acknowledgement.h b/lib/librdkafka-2.15.0/src/rdkafka_share_acknowledgement.h new file mode 100644 index 00000000000..fd55f7eed6a --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdkafka_share_acknowledgement.h @@ -0,0 +1,373 @@ +/* + * librdkafka - The Apache Kafka C/C++ library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _RDKAFKA_SHARE_ACKNOWLEDGEMENT_H_ +#define _RDKAFKA_SHARE_ACKNOWLEDGEMENT_H_ + +/* Forward declarations */ +typedef struct rd_kafka_op_s rd_kafka_op_t; +typedef struct rd_kafka_broker_s rd_kafka_broker_t; +typedef struct rd_kafka_cgrp_s rd_kafka_cgrp_t; + +typedef enum rd_kafka_internal_ShareAcknowledgement_type_s { + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED = + -1, /* Acquired records, not acknowledged yet */ + RD_KAFKA_SHARE_INTERNAL_ACK_GAP = 0, /* gap */ + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT = 1, /* accept */ + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE = 2, /* release */ + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT = 3 /* reject */ +} rd_kafka_share_internal_acknowledgement_type; + +/** + * @brief Acknowledgement batch entry for a contiguous offset range. + * + * Tracks acknowledgement status for each offset in the range. + * Used for building ShareAcknowledge requests. + * + * The size field always represents the number of offsets in the range + * (end_offset - start_offset + 1). + * + * The types_cnt field represents the actual size of the types array: + * - For inflight tracking: types_cnt == size (one type per offset) + * - For collated batches: types_cnt == 1 (single consolidated type) + */ +/** + * TODO KIP-932: Check naming. + */ +typedef struct rd_kafka_share_ack_batch_entry_s { + int64_t start_offset; /**< First offset in range */ + int64_t end_offset; /**< Last offset in range (inclusive) */ + int64_t size; /**< Number of offsets (end - start + 1) */ + int32_t types_cnt; /**< Number of elements in types array */ + int16_t delivery_count; /**< From AcquiredRecords DeliveryCount */ + rd_kafka_share_internal_acknowledgement_type + *types; /**< Array of ack types */ +} rd_kafka_share_ack_batch_entry_t; + +/** + * @brief Per topic-partition inflight acknowledgement batches. + * + * Tracks all acquired records for a topic-partition that are + * pending acknowledgement from the application. + * + * The rktpar field contains topic, partition, and in its _private: + * - rktp (toppar reference, refcount held) + * - topic_id + * + * response_leader_id is the broker id from which the records were + * acquired and is used to send the acknowledgement back to the same + * broker. It may differ from the current leader when sending + * acknowledgements. + */ +/** + * TODO KIP-932: Check naming. + */ +typedef struct rd_kafka_share_ack_batches_s { + rd_kafka_topic_partition_t + *rktpar; /**< Topic-partition with rktp ref */ + int32_t response_leader_id; /**< Leader broker id when records + * were acquired */ + int64_t response_acquired_offsets_count; /**< Total acquired messages */ + rd_list_t entries; /**< rd_kafka_share_ack_batch_entry_t*, + * sorted by start_offset */ +} rd_kafka_share_ack_batches_t; + +/** Allocate and initialize a share ack batch entry (offset range + types + * array). */ +rd_kafka_share_ack_batch_entry_t * +rd_kafka_share_ack_batch_entry_new(int64_t start_offset, + int64_t end_offset, + int32_t types_cnt, + int16_t delivery_count); +/** Destroy a share ack batch entry (frees types array and entry). */ +void rd_kafka_share_ack_batch_entry_destroy( + rd_kafka_share_ack_batch_entry_t *entry); + +/** Deep copy a share ack batch entry. */ +rd_kafka_share_ack_batch_entry_t *rd_kafka_share_ack_batch_entry_copy( + const rd_kafka_share_ack_batch_entry_t *src); + +/** Allocate and initialize a share ack batches. + * Takes ownership of \p rktpar. + * TODO KIP-932: We should keep a copy of the rktpar instead of taking + * ownership. */ +rd_kafka_share_ack_batches_t * +rd_kafka_share_ack_batches_new(rd_kafka_topic_partition_t *rktpar, + int32_t response_leader_id, + int64_t response_acquired_offsets_count); + +/** Allocate an empty share ack batches (all fields zeroed). */ +rd_kafka_share_ack_batches_t *rd_kafka_share_ack_batches_new_empty(void); + +/** void* wrapper for rd_kafka_share_ack_batches_destroy. + * Suitable as rd_list_t / rd_map_t value destructor. */ +void rd_kafka_share_ack_batches_destroy_free(void *ptr); + +/** Destroy share ack batches (frees entries, rktpar, and struct). */ +void rd_kafka_share_ack_batches_destroy(rd_kafka_share_ack_batches_t *batches); + +/** Deep copy a share ack batches (copies rktpar and all entries). */ +rd_kafka_share_ack_batches_t * +rd_kafka_share_ack_batches_copy(const rd_kafka_share_ack_batches_t *src); + +/** void* wrapper for rd_kafka_share_ack_batches_copy. + * Suitable as rd_list_copy_to callback. */ +void *rd_kafka_share_ack_batches_copy_void(const void *elem, void *opaque); + +/** + * @brief Transfer inflight acks from response RKO into rkshare's inflight map. + */ +void rd_kafka_share_build_inflight_acks_map(rd_kafka_share_t *rkshare, + rd_kafka_op_t *response_rko); + + +/** + * @brief Implicit ack: convert all ACQUIRED types to ACCEPT in inflight map. + */ +void rd_kafka_share_ack_all(rd_kafka_share_t *rkshare); + +rd_kafka_share_ack_batches_t * +rd_kafka_share_find_ack_batch(rd_list_t *ack_list, + const rd_kafka_topic_partition_t *rktpar); + +/** + * @brief Find an existing ack batch by topic id and partition. + * + * TODO KIP-932: Merge with rd_kafka_share_find_ack_batch and use + * binary search once ack_list is kept sorted. + */ +rd_kafka_share_ack_batches_t * +rd_kafka_share_find_ack_batch_by_id(rd_list_t *ack_list, + rd_kafka_Uuid_t topic_id, + int32_t partition); + +/** + * @brief Reply to a SHARE_FETCH op, propagating any top-level error + * to each batch in ack_details. + * + * On top-level error (err != 0), sets batch->rktpar->err = err on + * each batch in rko->rko_u.share_fetch.ack_details since the + * partition-level data was not parsed. On success (err == 0), the + * per-partition errors have already been set by the response parser + * and ack_details is left untouched. Always calls rd_kafka_op_reply + * at the end. Safe to use as a drop-in replacement for + * rd_kafka_op_reply on SHARE_FETCH ops. + */ +void rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err( + rd_kafka_op_t *rko, + rd_kafka_resp_err_t err); + +void rd_kafka_share_segregate_acks_by_leader(rd_kafka_t *rk, + rd_list_t *ack_batches); + +/** + * @brief Segregate sync ack batches by partition leader into each + * broker's pending_commit_sync list. + * + * @locality main thread + */ +void rd_kafka_share_segregate_sync_acks_by_leader(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg, + rd_list_t *ack_batches, + rd_ts_t abs_timeout); + +/** + * @brief Extract acknowledged (non-ACQUIRED) records from inflight map. + * + * Non-ACQUIRED offsets are collated into ack_details for sending. + * ACQUIRED offsets remain in the map. Empty entries are removed. + * + * @returns Allocated list or NULL if nothing to send. Caller must destroy. + */ +rd_list_t *rd_kafka_share_build_ack_details(rd_kafka_share_t *rkshare); + +/** + * @brief Check if share consumer uses implicit acknowledgement mode. + */ +rd_bool_t +rd_kafka_share_acknowledgement_mode_is_implicit(rd_kafka_share_t *rkshare); + +/** + * @brief Check if share consumer uses explicit acknowledgement mode. + */ +rd_bool_t +rd_kafka_share_acknowledgement_mode_is_explicit(rd_kafka_share_t *rkshare); + +/** + * @brief In implicit mode, acknowledge all acquired records as ACCEPT. + */ +void rd_kafka_share_acknowledge_all_if_implicit(rd_kafka_share_t *rkshare); + +/** + * @brief In explicit mode, ensure all acquired records have been acknowledged. + * @returns Error if there are unacknowledged records, NULL otherwise. + */ +rd_kafka_error_t * +rd_kafka_share_ensure_all_acknowledged_if_explicit(rd_kafka_share_t *rkshare); + +/** + * @brief Comparator for sorting/checking entries by start_offset. + * + * Used with rd_list_is_sorted(). + */ +int rd_kafka_share_ack_entries_sort_cmp_ptr(const void *_a, const void *_b); + +/** + * @struct rd_kafka_share_partition_offsets_s + * @brief Partition with set of acknowledged offsets. + */ +struct rd_kafka_share_partition_offsets_s { + rd_kafka_topic_partition_t + *partition; /**< Topic partition information */ + size_t cnt; /**< Number of offsets in array */ + int64_t offsets[]; /**< Flexible array of acknowledged offsets */ +}; + +/** + * @struct rd_kafka_share_partition_offsets_list_s + * @brief List of share partition offsets for callback. + */ +struct rd_kafka_share_partition_offsets_list_s { + size_t cnt; /**< Number of partitions */ + rd_kafka_share_partition_offsets_t *elems[]; /**< Flexible array of + pointers to partition + offsets */ +}; + + +/** + * @brief Destroy a partition offsets list. + * + * Frees all elements and the list itself. + * + * @param list List to destroy (may be NULL). + */ +void rd_kafka_share_partition_offsets_list_destroy( + rd_kafka_share_partition_offsets_list_t *list); + +/** + * @brief Build partition offsets list for a single partition. + * + * Creates an rd_kafka_share_partition_offsets_list_t with exactly one element + * for the given batches. Used for per-partition callback invocation. + * + * @param batches Single partition's ack batches. + * @returns Allocated list with one element, or NULL if no offsets. + * Caller must destroy with + * rd_kafka_share_partition_offsets_list_destroy(). + */ +rd_kafka_share_partition_offsets_list_t * +rd_kafka_share_build_partition_offsets_list( + rd_kafka_share_ack_batches_t *batches); + +/** + * @brief Enqueue share acknowledgement callback for a single partition. + * + * Creates and enqueues a callback op for the given partition with the + * specified error code. Used for both per-partition acknowledgement results + * and top-level errors. + * + * @param rk Kafka handle. + * @param batches The ack batches for this partition (contains offsets). + * @param err Error code to report in callback. + */ +void rd_kafka_share_enqueue_ack_commit_cb_op( + rd_kafka_t *rk, + rd_kafka_share_ack_batches_t *batches, + rd_kafka_resp_err_t err); + + +/** + * @brief Enqueue acknowledgement callbacks to application for each partition. + * + * Iterates through each partition in ack_details and enqueues one callback + * operation (RD_KAFKA_OP_SHARE_ACK_COMMIT_CB_EXECUTE) per partition to the + * application's reply queue. Each operation contains: + * - The partition's acknowledged offsets + * - Per-partition error code from batch->rktpar->err + * + * The application's runtime acknowledgement callback (set via + * rd_kafka_share_set_acknowledgement_commit_cb()) is invoked once per + * partition when the app calls rd_kafka_consumer_poll() or + * rd_kafka_queue_poll(). + * + * @param rk Kafka handle. + * @param ack_details List of rd_kafka_share_ack_batches_t* with acknowledgement + * results and per-partition error codes. + */ +void rd_kafka_share_dispatch_ack_callbacks(rd_kafka_t *rk, + rd_list_t *ack_details); + + +/** + * @brief Clear cached share-ack state on a broker that is being + * decommissioned. + * + * For each pending async ack batch on the broker, stamps + * SHARE_SESSION_NOT_FOUND on the batch and fires the share-ack + * callback so the application sees the failure. For sync acks + * (pending_commit_sync), stamps the same err and applies it to the + * in-flight commit_sync request (decrements awaiting count, completes + * sync if last broker outstanding). + * + * @locality main thread. + */ +void rd_kafka_share_acks_clear_during_broker_decommission( + rd_kafka_t *rk, + rd_kafka_broker_t *rkb); + + +/** + * @brief Apply a broker's commit_sync result: copy each batch's + * per-partition err onto the corresponding entry in + * rkcg_commit_sync_request.results, decrement the count of + * brokers still awaiting reply, and complete the commit_sync + * if this was the last broker outstanding. + * + * @locality main thread. + */ +void rd_kafka_share_commit_sync_apply_result(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg, + rd_list_t *ack_batches); + +/** + * @brief Check if all broker results are in and send response if done. + * + * @locality main thread. + */ +void rd_kafka_share_commit_sync_maybe_complete(rd_kafka_t *rk, + rd_kafka_cgrp_t *rkcg); + +/** + * @brief Send commit_sync response to the app thread and clear state. + * + * @locality main thread. + */ +void rd_kafka_share_commit_sync_send_response(rd_kafka_cgrp_t *rkcg); + + +#endif /* _RDKAFKA_SHARE_ACKNOWLEDGEMENT_H_ */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_ssl.c b/lib/librdkafka-2.15.0/src/rdkafka_ssl.c similarity index 88% rename from lib/librdkafka-2.10.1/src/rdkafka_ssl.c rename to lib/librdkafka-2.15.0/src/rdkafka_ssl.c index 417da81884f..35c1fac7f89 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_ssl.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_ssl.c @@ -36,6 +36,7 @@ #include "rdkafka_int.h" #include "rdkafka_transport_int.h" #include "rdkafka_cert.h" +#include "rdunittest.h" #ifdef _WIN32 #include @@ -135,11 +136,14 @@ const char *rd_kafka_ssl_last_error_str(void) { * * If 'rkb' is non-NULL broker-specific logging will be used, * else it will fall back on global 'rk' debugging. + * + * `ctx_identifier` is a string used to customize the log message. */ -static char *rd_kafka_ssl_error(rd_kafka_t *rk, - rd_kafka_broker_t *rkb, - char *errstr, - size_t errstr_size) { +char *rd_kafka_ssl_error0(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + const char *ctx_identifier, + char *errstr, + size_t errstr_size) { unsigned long l; const char *file, *data, *func; int line, flags; @@ -166,9 +170,11 @@ static char *rd_kafka_ssl_error(rd_kafka_t *rk, if (cnt++ > 0) { /* Log last message */ if (rkb) - rd_rkb_log(rkb, LOG_ERR, "SSL", "%s", errstr); + rd_rkb_log(rkb, LOG_ERR, "SSL", "%s: %s", + ctx_identifier, errstr); else - rd_kafka_log(rk, LOG_ERR, "SSL", "%s", errstr); + rd_kafka_log(rk, LOG_ERR, "SSL", "%s: %s", + ctx_identifier, errstr); } ERR_error_string_n(l, buf, sizeof(buf)); @@ -188,12 +194,18 @@ static char *rd_kafka_ssl_error(rd_kafka_t *rk, if (cnt == 0) rd_snprintf(errstr, errstr_size, - "No further error information available"); + "%s: No further error information available", + ctx_identifier); return errstr; } - +static char *rd_kafka_ssl_error(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + char *errstr, + size_t errstr_size) { + return rd_kafka_ssl_error0(rk, rkb, "kafka", errstr, errstr_size); +} /** * Set transport IO event polling based on SSL error. @@ -226,14 +238,12 @@ rd_kafka_transport_ssl_io_update(rd_kafka_transport_t *rktrans, rd_kafka_ssl_error(NULL, rktrans->rktrans_rkb, errstr, errstr_size); else if (!rd_socket_errno) { - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: connection closed by " - "peer"); - rd_snprintf(errstr, errstr_size, "Disconnected"); + rd_snprintf(errstr, errstr_size, + "Disconnected: connection closed by " + "peer"); } else if (rd_socket_errno == ECONNRESET) { - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: connection reset by peer"); - rd_snprintf(errstr, errstr_size, "Disconnected"); + rd_snprintf(errstr, errstr_size, + "Disconnected: connection reset by peer"); } else rd_snprintf(errstr, errstr_size, "SSL transport error: %s", @@ -241,9 +251,8 @@ rd_kafka_transport_ssl_io_update(rd_kafka_transport_t *rktrans, return -1; case SSL_ERROR_ZERO_RETURN: - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: SSL connection closed by peer"); - rd_snprintf(errstr, errstr_size, "Disconnected"); + rd_snprintf(errstr, errstr_size, + "Disconnected: SSL connection closed by peer"); return -1; default: @@ -441,6 +450,38 @@ static int rd_kafka_transport_ssl_cert_verify_cb(int preverify_ok, return 1; /* verification successful */ } +/** + * @brief Normalize hostname for SSL certificate verification. + * + * Strips trailing dot from hostname as X.509 certificates (per RFC 5280) + * don't include them in Subject Alternative Names (SANs). + * The trailing dot is used in DNS to indicate an absolute FQDN, + * but certificate SANs use a different representation without it. + * + * @param hostname Input hostname (may have trailing dot) + * @param normalized Output buffer for normalized hostname + * @param size Size of output buffer + * + * @returns The normalized hostname (same as \p normalized) + * + * @remark This function is exposed for testing via ENABLE_DEVEL. + */ +static const char *rd_kafka_ssl_normalize_hostname(const char *hostname, + char *normalized, + size_t size) { + size_t len; + + rd_snprintf(normalized, size, "%s", hostname); + len = strlen(normalized); + + /* Strip trailing dot (unless it's a single dot) */ + if (len > 1 && normalized[len - 1] == '.') { + normalized[len - 1] = '\0'; + } + + return normalized; +} + /** * @brief Set TLSEXT hostname for SNI and optionally enable * SSL endpoint identification verification. @@ -451,6 +492,7 @@ static int rd_kafka_transport_ssl_set_endpoint_id(rd_kafka_transport_t *rktrans, char *errstr, size_t errstr_size) { char name[RD_KAFKA_NODENAME_SIZE]; + char name_for_verify[RD_KAFKA_NODENAME_SIZE]; char *t; rd_kafka_broker_lock(rktrans->rktrans_rkb); @@ -462,13 +504,21 @@ static int rd_kafka_transport_ssl_set_endpoint_id(rd_kafka_transport_t *rktrans, if ((t = strrchr(name, ':'))) *t = '\0'; + /* Normalize hostname (remove trailing dot) for both SNI and certificate + * verification */ + rd_kafka_ssl_normalize_hostname(name, name_for_verify, + sizeof(name_for_verify)); + #if (OPENSSL_VERSION_NUMBER >= 0x0090806fL) && !defined(OPENSSL_NO_TLSEXT) /* If non-numerical hostname, send it for SNI */ - if (!(/*ipv6*/ (strchr(name, ':') && - strspn(name, "0123456789abcdefABCDEF:.[]%") == - strlen(name)) || - /*ipv4*/ strspn(name, "0123456789.") == strlen(name)) && - !SSL_set_tlsext_host_name(rktrans->rktrans_ssl, name)) + if (!(/*ipv6*/ ( + strchr(name_for_verify, ':') && + strspn(name_for_verify, "0123456789abcdefABCDEF:.[]%") == + strlen(name_for_verify)) || + /*ipv4*/ + strspn(name_for_verify, "0123456789.") == + strlen(name_for_verify)) && + !SSL_set_tlsext_host_name(rktrans->rktrans_ssl, name_for_verify)) goto fail; #endif @@ -476,8 +526,16 @@ static int rd_kafka_transport_ssl_set_endpoint_id(rd_kafka_transport_t *rktrans, RD_KAFKA_SSL_ENDPOINT_ID_NONE) return 0; + /* Log if we stripped a trailing dot */ + if (strcmp(name, name_for_verify) != 0) { + rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "ENDPOINT", + "Stripped trailing dot from hostname for " + "certificate verification: %s -> %s", + name, name_for_verify); + } + #if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(OPENSSL_IS_BORINGSSL) - if (!SSL_set1_host(rktrans->rktrans_ssl, name)) + if (!SSL_set1_host(rktrans->rktrans_ssl, name_for_verify)) goto fail; #elif OPENSSL_VERSION_NUMBER >= 0x1000200fL /* 1.0.2 */ { @@ -485,8 +543,9 @@ static int rd_kafka_transport_ssl_set_endpoint_id(rd_kafka_transport_t *rktrans, param = SSL_get0_param(rktrans->rktrans_ssl); - if (!X509_VERIFY_PARAM_set1_host(param, name, - strnlen(name, sizeof(name)))) + if (!X509_VERIFY_PARAM_set1_host( + param, name_for_verify, + strnlen(name_for_verify, sizeof(name_for_verify)))) goto fail; } #else @@ -498,7 +557,8 @@ static int rd_kafka_transport_ssl_set_endpoint_id(rd_kafka_transport_t *rktrans, #endif rd_rkb_dbg(rktrans->rktrans_rkb, SECURITY, "ENDPOINT", - "Enabled endpoint identification using hostname %s", name); + "Enabled endpoint identification using hostname %s", + name_for_verify); return 0; @@ -665,7 +725,7 @@ int rd_kafka_transport_ssl_handshake(rd_kafka_transport_t *rktrans) { " (install ca-certificates package)" #endif ; - else if (!strcmp(errstr, "Disconnected")) { + else if (rd_kafka_transport_error_disconnected(errstr)) { extra = ": connecting to a PLAINTEXT broker listener?"; /* Disconnects during handshake are most likely * not due to SSL, but rather at the transport level */ @@ -802,6 +862,7 @@ static X509 *rd_kafka_ssl_X509_from_string(rd_kafka_t *rk, * @brief Attempt load CA certificates from a Windows Certificate store. */ static int rd_kafka_ssl_win_load_cert_store(rd_kafka_t *rk, + const char *ctx_identifier, SSL_CTX *ctx, const char *store_name) { HCERTSTORE w_store; @@ -816,10 +877,11 @@ static int rd_kafka_ssl_win_load_cert_store(rd_kafka_t *rk, /* Convert store_name to wide-char */ werr = mbstowcs_s(&wsize, NULL, 0, store_name, strlen(store_name)); if (werr || wsize < 2 || wsize > 1000) { - rd_kafka_log(rk, LOG_ERR, "CERTSTORE", - "Invalid Windows certificate store name: %.*s%s", - 30, store_name, - wsize < 2 ? " (empty)" : " (truncated)"); + rd_kafka_log( + rk, LOG_ERR, "CERTSTORE", + "%s: Invalid Windows certificate store name: %.*s%s", + ctx_identifier, 30, store_name, + wsize < 2 ? " (empty)" : " (truncated)"); return -1; } wstore_name = rd_alloca(sizeof(*wstore_name) * wsize); @@ -835,9 +897,9 @@ static int rd_kafka_ssl_win_load_cert_store(rd_kafka_t *rk, if (!w_store) { rd_kafka_log( rk, LOG_ERR, "CERTSTORE", - "Failed to open Windows certificate " + "%s: Failed to open Windows certificate " "%s store: %s", - store_name, + ctx_identifier, store_name, rd_strerror_w32(GetLastError(), errstr, sizeof(errstr))); return -1; } @@ -873,9 +935,9 @@ static int rd_kafka_ssl_win_load_cert_store(rd_kafka_t *rk, CertCloseStore(w_store, 0); rd_kafka_dbg(rk, SECURITY, "CERTSTORE", - "%d certificate(s) successfully added from " + "%s: %d certificate(s) successfully added from " "Windows Certificate %s store, %d failed", - cnt, store_name, fail_cnt); + ctx_identifier, cnt, store_name, fail_cnt); if (cnt == 0 && fail_cnt > 0) return -1; @@ -888,9 +950,10 @@ static int rd_kafka_ssl_win_load_cert_store(rd_kafka_t *rk, * * @returns the number of successfully loaded certificates, or -1 on error. */ -static int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk, - SSL_CTX *ctx, - const char *store_names) { +int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk, + const char *ctx_identifier, + SSL_CTX *ctx, + const char *store_names) { char *s; int cert_cnt = 0, fail_cnt = 0; @@ -924,7 +987,8 @@ static int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk, s = ""; } - r = rd_kafka_ssl_win_load_cert_store(rk, ctx, store_name); + r = rd_kafka_ssl_win_load_cert_store(rk, ctx_identifier, ctx, + store_name); if (r != -1) cert_cnt += r; else @@ -938,7 +1002,32 @@ static int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk, } #endif /* MSC_VER */ +/** + * @brief Probe for a single \p path and if found and not an empty directory, + * set it on the \p ctx. + * + * @returns 0 if CA location was set with an error, 1 if it was set correctly, + * -1 if path should be skipped. + */ +static int rd_kafka_ssl_set_ca_path(rd_kafka_t *rk, + const char *ctx_identifier, + const char *path, + SSL_CTX *ctx, + rd_bool_t *is_dir) { + if (!rd_file_stat(path, is_dir)) + return -1; + + if (*is_dir && rd_kafka_dir_is_empty(path)) + return -1; + + rd_kafka_dbg(rk, SECURITY, "CACERTS", + "Setting default CA certificate location for %s " + "to \"%s\"", + ctx_identifier, path); + return SSL_CTX_load_verify_locations(ctx, *is_dir ? NULL : path, + *is_dir ? path : NULL); +} /** * @brief Probe for the system's CA certificate location and if found set it @@ -946,8 +1035,9 @@ static int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk, * * @returns 0 if CA location was set, else -1. */ -static int rd_kafka_ssl_probe_and_set_default_ca_location(rd_kafka_t *rk, - SSL_CTX *ctx) { +int rd_kafka_ssl_probe_and_set_default_ca_location(rd_kafka_t *rk, + const char *ctx_identifier, + SSL_CTX *ctx) { #if _WIN32 /* No standard location on Windows, CA certs are in the ROOT store. */ return -1; @@ -999,34 +1089,21 @@ static int rd_kafka_ssl_probe_and_set_default_ca_location(rd_kafka_t *rk, int i; for (i = 0; (path = paths[i]); i++) { - struct stat st; rd_bool_t is_dir; - int r; - - if (stat(path, &st) != 0) - continue; - - is_dir = S_ISDIR(st.st_mode); - - if (is_dir && rd_kafka_dir_is_empty(path)) + int r = rd_kafka_ssl_set_ca_path(rk, ctx_identifier, path, ctx, + &is_dir); + if (r == -1) continue; - rd_kafka_dbg(rk, SECURITY, "CACERTS", - "Setting default CA certificate location " - "to %s, override with ssl.ca.location", - path); - - r = SSL_CTX_load_verify_locations(ctx, is_dir ? NULL : path, - is_dir ? path : NULL); if (r != 1) { char errstr[512]; /* Read error and clear the error stack */ rd_kafka_ssl_error(rk, NULL, errstr, sizeof(errstr)); rd_kafka_dbg(rk, SECURITY, "CACERTS", "Failed to set default CA certificate " - "location to %s %s: %s: skipping", + "location to %s %s for %s: %s: skipping", is_dir ? "directory" : "file", path, - errstr); + ctx_identifier, errstr); continue; } @@ -1035,7 +1112,8 @@ static int rd_kafka_ssl_probe_and_set_default_ca_location(rd_kafka_t *rk, rd_kafka_dbg(rk, SECURITY, "CACERTS", "Unable to find any standard CA certificate" - "paths: is the ca-certificates package installed?"); + "paths for %s: is the ca-certificates package installed?", + ctx_identifier); return -1; #endif } @@ -1255,7 +1333,7 @@ static int rd_kafka_ssl_set_certs(rd_kafka_t *rk, /* Attempt to load CA root certificates from the * configured Windows certificate stores. */ r = rd_kafka_ssl_win_load_cert_stores( - rk, ctx, rk->rk_conf.ssl.ca_cert_stores); + rk, "kafka", ctx, rk->rk_conf.ssl.ca_cert_stores); if (r == 0) { rd_kafka_log( rk, LOG_NOTICE, "CERTSTORE", @@ -1286,8 +1364,8 @@ static int rd_kafka_ssl_set_certs(rd_kafka_t *rk, * of standard CA certificate paths and use the * first one that is found. * Ignore failures. */ - r = rd_kafka_ssl_probe_and_set_default_ca_location(rk, - ctx); + r = rd_kafka_ssl_probe_and_set_default_ca_location( + rk, "kafka", ctx); } if (r == -1) { @@ -1967,14 +2045,7 @@ rd_kafka_transport_ssl_lock_cb(int mode, int i, const char *file, int line) { #endif static RD_UNUSED unsigned long rd_kafka_transport_ssl_threadid_cb(void) { -#ifdef _WIN32 - /* Windows makes a distinction between thread handle - * and thread id, which means we can't use the - * thrd_current() API that returns the handle. */ - return (unsigned long)GetCurrentThreadId(); -#else - return (unsigned long)(intptr_t)thrd_current(); -#endif + return thrd_current_id(); } #ifdef HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK @@ -2101,3 +2172,64 @@ int rd_kafka_ssl_hmac(rd_kafka_broker_t *rkb, return 0; } + +/** + * @brief Unit test for SSL hostname normalization. + * + * Tests the rd_kafka_ssl_normalize_hostname() function with various edge cases + * to verify that trailing dots are correctly stripped from hostnames for + * SSL certificate verification. + */ +int unittest_ssl(void) { + int fails = 0; + struct { + const char *input; + const char *expected; + const char *description; + } test_cases[] = { + {"broker.example.com.", "broker.example.com", + "FQDN with trailing dot"}, + {"broker.example.com", "broker.example.com", + "FQDN without trailing dot"}, + {"localhost.", "localhost", "localhost with trailing dot"}, + {"localhost", "localhost", "localhost without trailing dot"}, + {".", ".", "single dot (edge case - should remain unchanged)"}, + {"", "", "empty string (edge case)"}, + {"broker-1.example.com.", "broker-1.example.com", + "hostname with dash and trailing dot"}, + {"192.168.1.1", "192.168.1.1", "IP address (no trailing dot)"}, + {NULL, NULL, NULL}}; + int i; + + RD_UT_SAY("Testing hostname normalization edge cases"); + + for (i = 0; test_cases[i].input != NULL; i++) { + char normalized[256]; + const char *input = test_cases[i].input; + const char *expected = test_cases[i].expected; + const char *desc = test_cases[i].description; + const char *result; + + /* Call the function under test */ + result = rd_kafka_ssl_normalize_hostname(input, normalized, + sizeof(normalized)); + + /* Verify the function returns the normalized buffer */ + RD_UT_ASSERT(result == normalized, + "Test case %d (%s): Function should return the " + "normalized buffer", + i + 1, desc); + + /* Verify the normalization is correct */ + RD_UT_ASSERT(!strcmp(result, expected), + "Test case %d (%s): Hostname normalization " + "failed: expected \"%s\" but got \"%s\"", + i + 1, desc, expected, result); + + RD_UT_SAY("Test case %d passed: %s", i + 1, desc); + } + + RD_UT_SAY("All %d hostname normalization edge cases passed", i); + + return fails; +} diff --git a/lib/librdkafka-2.10.1/src/rdkafka_ssl.h b/lib/librdkafka-2.15.0/src/rdkafka_ssl.h similarity index 81% rename from lib/librdkafka-2.10.1/src/rdkafka_ssl.h rename to lib/librdkafka-2.15.0/src/rdkafka_ssl.h index dd56177319a..6307857c1d6 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_ssl.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_ssl.h @@ -66,4 +66,21 @@ int rd_kafka_ssl_read_cert_chain_from_BIO(BIO *in, pem_password_cb *password_cb, void *password_cb_opaque); +int rd_kafka_ssl_probe_and_set_default_ca_location(rd_kafka_t *rk, + const char *ctx_identifier, + SSL_CTX *ctx); + +char *rd_kafka_ssl_error0(rd_kafka_t *rk, + rd_kafka_broker_t *rkb, + const char *ctx_identifier, + char *errstr, + size_t errstr_size); + +#ifdef _WIN32 +int rd_kafka_ssl_win_load_cert_stores(rd_kafka_t *rk, + const char *ctx_identifier, + SSL_CTX *ctx, + const char *store_names); +#endif + #endif /* _RDKAFKA_SSL_H_ */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_sticky_assignor.c b/lib/librdkafka-2.15.0/src/rdkafka_sticky_assignor.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_sticky_assignor.c rename to lib/librdkafka-2.15.0/src/rdkafka_sticky_assignor.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_subscription.c b/lib/librdkafka-2.15.0/src/rdkafka_subscription.c similarity index 64% rename from lib/librdkafka-2.10.1/src/rdkafka_subscription.c rename to lib/librdkafka-2.15.0/src/rdkafka_subscription.c index 46ab544ee20..2805129c0d8 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_subscription.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_subscription.c @@ -46,6 +46,31 @@ rd_kafka_resp_err_t rd_kafka_unsubscribe(rd_kafka_t *rk) { rd_kafka_op_req2(rkcg->rkcg_ops, RD_KAFKA_OP_SUBSCRIBE)); } +rd_kafka_resp_err_t rd_kafka_share_unsubscribe(rd_kafka_share_t *rkshare) { + rd_kafka_resp_err_t err; + rd_kafka_error_t *error = NULL; + + /** + * TODO KIP-932: Guard this with checks for rkshare and + * rkshare->rkshare_rk? + */ + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) { + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + return err; + } + + if (unlikely((err = rd_kafka_share_consumer_closed_err(rkshare)))) + goto done; + + err = rd_kafka_unsubscribe(rkshare->rkshare_rk); + if (!err) + rkshare->rkshare_subscribed = rd_false; +done: + rd_kafka_share_release(rkshare); + return err; +} + /** @returns 1 if the topic is invalid (bad regex, empty), else 0 if valid. */ static size_t _invalid_topic_cb(const rd_kafka_topic_partition_t *rktpar, @@ -68,6 +93,20 @@ static size_t _invalid_topic_cb(const rd_kafka_topic_partition_t *rktpar, } +/** @returns 1 if the topic is empty (invalid for the share consumer), + * else 0. Share consumer subscriptions are plain topic names: + * entries are forwarded to the broker verbatim and are never + * expanded as regex/wildcards, so a leading '^' is treated as a + * literal character rather than a pattern marker. */ +static size_t _share_invalid_topic_cb(const rd_kafka_topic_partition_t *rktpar, + void *opaque) { + if (!*rktpar->topic) + return 1; + + return 0; +} + + rd_kafka_resp_err_t rd_kafka_subscribe(rd_kafka_t *rk, const rd_kafka_topic_partition_list_t *topics) { @@ -98,6 +137,76 @@ rd_kafka_subscribe(rd_kafka_t *rk, rd_kafka_op_req(rkcg->rkcg_ops, rko, RD_POLL_INFINITE)); } +rd_kafka_resp_err_t +rd_kafka_share_subscribe(rd_kafka_share_t *rkshare, + const rd_kafka_topic_partition_list_t *topics) { + rd_kafka_op_t *rko; + rd_kafka_cgrp_t *rkcg; + rd_kafka_topic_partition_list_t *topics_cpy; + rd_kafka_resp_err_t err; + rd_kafka_error_t *error = NULL; + + /** + * TODO KIP-932: Guard this with checks for rkshare and + * rkshare->rkshare_rk? + */ + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) { + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + return err; + } + + if (unlikely((err = rd_kafka_share_consumer_closed_err(rkshare)))) + goto done; + + if (!(rkcg = rd_kafka_cgrp_get(rkshare->rkshare_rk))) { + err = RD_KAFKA_RESP_ERR__UNKNOWN_GROUP; + goto done; + } + + /* An empty topic list is equivalent to unsubscribe. */ + if (topics->cnt == 0) { + err = rd_kafka_share_unsubscribe(rkshare); + goto done; + } + + /* Share consumer subscriptions are plain topic names forwarded to + * the broker verbatim; entries are never expanded as regex/wildcards + * (a leading '^' is a literal character, not a pattern marker). + * Reject only empty entries; pass everything else through via the + * heartbeat. */ + if (rd_kafka_topic_partition_list_sum(topics, _share_invalid_topic_cb, + NULL) > 0) { + err = RD_KAFKA_RESP_ERR__INVALID_ARG; + goto done; + } + + topics_cpy = rd_kafka_topic_partition_list_copy(topics); + if (rd_kafka_topic_partition_list_has_duplicates( + topics_cpy, rd_true /*ignore partition field*/)) { + rd_kafka_topic_partition_list_destroy(topics_cpy); + err = RD_KAFKA_RESP_ERR__INVALID_ARG; + goto done; + } + + rko = rd_kafka_op_new(RD_KAFKA_OP_SUBSCRIBE); + rko->rko_u.subscribe.topics = topics_cpy; + + err = rd_kafka_op_err_destroy( + rd_kafka_op_req(rkcg->rkcg_ops, rko, RD_POLL_INFINITE)); + /** + * TODO KIP-932: It can only return FATAL error from the main thread. + * In which case it unsubscribes. Check if we should change the flag + * to false when FATAL error is received or keep it as true which + * will let the user go through the normal consumer flow. + */ + if (!err) + rkshare->rkshare_subscribed = rd_true; +done: + rd_kafka_share_release(rkshare); + return err; +} + rd_kafka_error_t * rd_kafka_assign0(rd_kafka_t *rk, @@ -259,6 +368,30 @@ rd_kafka_subscription(rd_kafka_t *rk, return err; } +rd_kafka_resp_err_t +rd_kafka_share_subscription(rd_kafka_share_t *rkshare, + rd_kafka_topic_partition_list_t **topics) { + rd_kafka_resp_err_t err; + rd_kafka_error_t *error = NULL; + + /** + * TODO KIP-932: Guard this with checks for rkshare and + * rkshare->rkshare_rk? + */ + if (unlikely((error = rd_kafka_share_acquire(rkshare)) != NULL)) { + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + return err; + } + + if (unlikely((err = rd_kafka_share_consumer_closed_err(rkshare)))) + goto done; + + err = rd_kafka_subscription(rkshare->rkshare_rk, topics); +done: + rd_kafka_share_release(rkshare); + return err; +} rd_kafka_resp_err_t rd_kafka_pause_partitions(rd_kafka_t *rk, diff --git a/lib/librdkafka-2.10.1/src/rdkafka_telemetry.c b/lib/librdkafka-2.15.0/src/rdkafka_telemetry.c similarity index 98% rename from lib/librdkafka-2.10.1/src/rdkafka_telemetry.c rename to lib/librdkafka-2.15.0/src/rdkafka_telemetry.c index 1c3e55d46cc..95dec818ff3 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_telemetry.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_telemetry.c @@ -52,10 +52,8 @@ static int rd_kafka_filter_broker_by_GetTelemetrySubscription(rd_kafka_broker_t *rkb, void *opaque) { - int features; - if (rd_kafka_broker_ApiVersion_supported0( - rkb, RD_KAFKAP_GetTelemetrySubscriptions, 0, 0, &features, - rd_false) != -1) + if (rd_kafka_broker_ApiVersion_at_least_no_lock( + rkb, RD_KAFKAP_GetTelemetrySubscriptions, 0)) return 0; return 1; } @@ -175,8 +173,9 @@ static void update_matched_metrics(rd_kafka_t *rk, size_t j) { static void rd_kafka_match_requested_metrics(rd_kafka_t *rk) { size_t metrics_cnt = RD_KAFKA_TELEMETRY_METRIC_CNT(rk), i; rd_bool_t is_metric_included[RD_MAX( - (int)RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT, - (int)RD_KAFKA_TELEMETRY_CONSUMER_METRIC__CNT)] = {0}; + RD_MAX((int)RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT, + (int)RD_KAFKA_TELEMETRY_CONSUMER_METRIC__CNT), + (int)RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRIC__CNT)] = {0}; const rd_kafka_telemetry_metric_info_t *info = RD_KAFKA_TELEMETRY_METRIC_INFO(rk); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_telemetry.h b/lib/librdkafka-2.15.0/src/rdkafka_telemetry.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_telemetry.h rename to lib/librdkafka-2.15.0/src/rdkafka_telemetry.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_decode.c b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_decode.c similarity index 79% rename from lib/librdkafka-2.10.1/src/rdkafka_telemetry_decode.c rename to lib/librdkafka-2.15.0/src/rdkafka_telemetry_decode.c index 452e43c3782..084f08c4764 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_decode.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_decode.c @@ -546,7 +546,14 @@ int unit_test_telemetry(rd_kafka_type_t rk_type, rk->rk_conf = *conf; rd_free(conf); - rk->rk_type = rk_type; + rk->rk_type = rk_type; + /* Auto-detect share consumer from metric name prefix. */ + if (strncmp(expected_name, + RD_KAFKA_TELEMETRY_METRIC_PREFIX "consumer.share.", + strlen(RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.")) == 0) + rk->rk_conf.share.is_share_consumer = 1; + rk->rk_cgrp = rd_calloc(1, sizeof(*rk->rk_cgrp)); rk->rk_broker_cnt.val = 1; rk->rk_telemetry.matched_metrics_cnt = 1; @@ -561,6 +568,12 @@ int unit_test_telemetry(rd_kafka_type_t rk_type, rd_avg_init(&rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_avg_init( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_poll_idle_ratio, + RD_AVG_GAUGE, 0, 1000 * 1000, 2, rd_true); + rd_avg_init( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_time_between_poll, + RD_AVG_GAUGE, 0, 60 * 1000 * 1000, 2, rd_true); rd_avg_init(&rk->rk_telemetry.rd_avg_current.rk_avg_commit_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); rd_avg_init(&rk->rk_telemetry.rd_avg_current.rk_avg_rebalance_latency, @@ -568,11 +581,21 @@ int unit_test_telemetry(rd_kafka_type_t rk_type, rd_avg_init(&rk->rk_telemetry.rd_avg_rollover.rk_avg_poll_idle_ratio, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_avg_init( + &rk->rk_telemetry.rd_avg_rollover.rk_avg_share_poll_idle_ratio, + RD_AVG_GAUGE, 0, 1000 * 1000, 2, rd_true); + rd_avg_init( + &rk->rk_telemetry.rd_avg_rollover.rk_avg_share_time_between_poll, + RD_AVG_GAUGE, 0, 60 * 1000 * 1000, 2, rd_true); rd_avg_init(&rk->rk_telemetry.rd_avg_rollover.rk_avg_commit_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); rd_avg_init(&rk->rk_telemetry.rd_avg_rollover.rk_avg_rebalance_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_atomic64_init(&rk->rk_telemetry.share_fetch_total, 0); + rd_atomic64_init(&rk->rk_telemetry.acknowledgements_send_total, 0); + rd_atomic64_init(&rk->rk_telemetry.heartbeat_total, 0); + rd_strlcpy(rk->rk_name, "unittest", sizeof(rk->rk_name)); clear_unit_test_data(expected_value_int, expected_value_double); @@ -600,6 +623,11 @@ int unit_test_telemetry(rd_kafka_type_t rk_type, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); rd_avg_init(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_fetch_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_avg_init( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_avg_init(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size, + RD_AVG_GAUGE, 0, 100 * 1024 * 1024, 2, rd_true); rd_avg_init(&rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_rtt, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); @@ -609,6 +637,12 @@ int unit_test_telemetry(rd_kafka_type_t rk_type, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); rd_avg_init(&rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_fetch_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_avg_init( + &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_share_fetch_latency, + RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); + rd_avg_init( + &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_share_fetch_size, + RD_AVG_GAUGE, 0, 100 * 1024 * 1024, 2, rd_true); rd_avg_init(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_produce_latency, RD_AVG_GAUGE, 0, 500 * 1000, 2, rd_true); rd_avg_init(&rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_produce_latency, @@ -704,10 +738,30 @@ int unit_test_telemetry(rd_kafka_type_t rk_type, rd_avg_destroy( &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_fetch_latency); + rd_avg_destroy( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency); + rd_avg_destroy( + &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_share_fetch_latency); + + rd_avg_destroy( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size); + rd_avg_destroy( + &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_share_fetch_size); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio); rd_avg_destroy( &rk->rk_telemetry.rd_avg_rollover.rk_avg_poll_idle_ratio); + rd_avg_destroy( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_poll_idle_ratio); + rd_avg_destroy( + &rk->rk_telemetry.rd_avg_rollover.rk_avg_share_poll_idle_ratio); + + rd_avg_destroy( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_time_between_poll); + rd_avg_destroy( + &rk->rk_telemetry.rd_avg_rollover.rk_avg_share_time_between_poll); + rd_avg_destroy( &rk->rk_telemetry.rd_avg_current.rk_avg_rebalance_latency); rd_avg_destroy( @@ -791,6 +845,29 @@ void unit_test_telemetry_set_fetch_latency(rd_kafka_t *rk, 1000); } +void unit_test_telemetry_set_share_fetch_latency(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_avg_add( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency, + 7000); + rd_avg_add( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency, + 15000); + rd_avg_add( + &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency, + 23000); +} + +void unit_test_telemetry_set_share_fetch_size(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_avg_add(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size, + 1024); + rd_avg_add(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size, + 2048); + rd_avg_add(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size, + 3072); +} + void unit_test_telemetry_set_poll_idle_ratio(rd_kafka_t *rk, rd_kafka_broker_t *rkb) { rd_avg_add(&rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio, @@ -801,6 +878,54 @@ void unit_test_telemetry_set_poll_idle_ratio(rd_kafka_t *rk, 1000000); } +void unit_test_telemetry_set_share_poll_idle_ratio(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_poll_idle_ratio, + 250000); + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_poll_idle_ratio, + 500000); + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_poll_idle_ratio, + 750000); +} + +void unit_test_telemetry_set_time_between_poll(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_time_between_poll, + 5000); + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_time_between_poll, + 25000); + rd_avg_add( + &rk->rk_telemetry.rd_avg_current.rk_avg_share_time_between_poll, + 60000); +} + +void unit_test_telemetry_set_share_fetch_total(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_atomic64_add(&rk->rk_telemetry.share_fetch_total, 10); + rd_atomic64_add(&rk->rk_telemetry.share_fetch_total, 15); + rd_atomic64_add(&rk->rk_telemetry.share_fetch_total, 17); +} + +void unit_test_telemetry_set_acknowledgements_send_total( + rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_atomic64_add(&rk->rk_telemetry.acknowledgements_send_total, 10); + rd_atomic64_add(&rk->rk_telemetry.acknowledgements_send_total, 15); + rd_atomic64_add(&rk->rk_telemetry.acknowledgements_send_total, 17); +} + +void unit_test_telemetry_set_heartbeat_total(rd_kafka_t *rk, + rd_kafka_broker_t *rkb) { + rd_atomic64_add(&rk->rk_telemetry.heartbeat_total, 10); + rd_atomic64_add(&rk->rk_telemetry.heartbeat_total, 15); + rd_atomic64_add(&rk->rk_telemetry.heartbeat_total, 17); +} + void unit_test_telemetry_set_commit_latency(rd_kafka_t *rk, rd_kafka_broker_t *rkb) { rd_avg_add(&rk->rk_telemetry.rd_avg_current.rk_avg_commit_latency, @@ -993,6 +1118,109 @@ int unit_test_telemetry_gauge(void) { RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_false, rd_false, unit_test_telemetry_set_commit_latency, default_expected_value_int, default_expected_value_double); + + /* Share consumer metrics */ + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_POLL_IDLE_RATIO_AVG, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.poll.idle.ratio.avg", + "The average fraction of time the consumer's poll() is idle " + "as opposed to waiting for the user code to process records.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_share_poll_idle_ratio, 0, 0.5); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_AVG, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.time.between.poll.avg", + "The average delay between invocations of poll() in " + "milliseconds.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_time_between_poll, 0, 30.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_MAX, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.time.between.poll.max", + "The max delay between invocations of poll() in milliseconds.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_false, rd_false, + unit_test_telemetry_set_time_between_poll, 60, 0.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_RATE, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.rate", + "The number of fetch requests per second.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_share_fetch_total, 0, 42.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_AVG, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.latency.avg", + "The average time taken for a fetch request.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_share_fetch_latency, 0, 15.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_MAX, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.latency.max", + "The maximum time taken for any fetch request.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_false, rd_false, + unit_test_telemetry_set_share_fetch_latency, 23, 0.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_RATE, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.acknowledgements.send.rate", + "The average number of record acknowledgements sent per second.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_acknowledgements_send_total, 0, 42.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_AVG, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.throttle.time.avg", + "The average throttle time in ms.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_throttle_time, default_expected_value_int, + default_expected_value_double); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_MAX, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.throttle.time.max", + "The maximum throttle time in ms.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_false, rd_false, + unit_test_telemetry_set_throttle_time, default_expected_value_int, + default_expected_value_double); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_RATE, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.coordinator.heartbeat.rate", + "The number of heartbeats per second.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_heartbeat_total, 0, 42.0); + /* Samples (1024 + 2048 + 3072) / 3 = 2048 */ + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_AVG, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.size.avg", + "The average number of bytes fetched per request.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_true, rd_false, + unit_test_telemetry_set_share_fetch_size, 0, 2048.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_MAX, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.size.max", + "The maximum number of bytes fetched per request.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE, rd_false, rd_false, + unit_test_telemetry_set_share_fetch_size, 3072, 0.0); return fails; } @@ -1042,6 +1270,32 @@ int unit_test_telemetry_sum(void) { RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM, rd_false, rd_false, unit_test_telemetry_set_rebalance_latency, default_expected_value_int, default_expected_value_double); + + /* Expected 42 */ + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.fetch.total", + "The total number of fetch requests.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM, rd_false, rd_false, + unit_test_telemetry_set_share_fetch_total, 42, 0.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.fetch.manager.acknowledgements.send.total", + "The total number of record acknowledgements sent.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM, rd_false, rd_false, + unit_test_telemetry_set_acknowledgements_send_total, 42, 0.0); + fails += unit_test_telemetry( + RD_KAFKA_CONSUMER, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_PREFIX + "consumer.share.coordinator.heartbeat.total", + "The total number of heartbeats.", + RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM, rd_false, rd_false, + unit_test_telemetry_set_heartbeat_total, 42, 0.0); return fails; } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_decode.h b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_decode.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_telemetry_decode.h rename to lib/librdkafka-2.15.0/src/rdkafka_telemetry_decode.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_encode.c b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_encode.c similarity index 71% rename from lib/librdkafka-2.10.1/src/rdkafka_telemetry_encode.c rename to lib/librdkafka-2.15.0/src/rdkafka_telemetry_encode.c index da1a341d8a1..0519bdb3f70 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_encode.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_encode.c @@ -307,6 +307,225 @@ calculate_consumer_commit_latency_max(rd_kafka_t *rk, return max_commit_time; } +static rd_kafka_telemetry_metric_value_t +calculate_share_consumer_poll_idle_ratio_avg(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t share_poll_idle_avg; + share_poll_idle_avg.double_value = calculate_avg( + rk->rk_telemetry.rd_avg_rollover.rk_avg_share_poll_idle_ratio, 1e6); + return share_poll_idle_avg; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_time_between_poll_avg(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t time_between_poll_avg; + time_between_poll_avg.double_value = calculate_avg( + rk->rk_telemetry.rd_avg_rollover.rk_avg_share_time_between_poll, + THREE_ORDERS_MAGNITUDE); + return time_between_poll_avg; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_time_between_poll_max(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t time_between_poll_max; + time_between_poll_max.int_value = calculate_max( + rk->rk_telemetry.rd_avg_rollover.rk_avg_share_time_between_poll, + THREE_ORDERS_MAGNITUDE); + return time_between_poll_max; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_fetch_latency_avg(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t avg_share_fetch_latency; + brokers_avg(rk, rkb_avg_share_fetch_latency, THREE_ORDERS_MAGNITUDE, + avg_share_fetch_latency); + return avg_share_fetch_latency; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_fetch_latency_max(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t max_share_fetch_latency; + brokers_max(rk, rkb_avg_share_fetch_latency, THREE_ORDERS_MAGNITUDE, + max_share_fetch_latency); + return max_share_fetch_latency; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_bytes_consumed_total(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t bytes_consumed_total; + int64_t current = + rd_atomic64_get(&rk->rk_telemetry.share_bytes_consumed_total); + + if (!rk->rk_telemetry.delta_temporality) + bytes_consumed_total.int_value = current; + else + bytes_consumed_total.int_value = + current - + rk->rk_telemetry.rk_historic_c.share_bytes_consumed_total; + + return bytes_consumed_total; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_bytes_consumed_rate(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t bytes_consumed_rate; + rd_ts_t ts_last = rk->rk_telemetry.rk_historic_c.ts_last; + int64_t delta = + rd_atomic64_get(&rk->rk_telemetry.share_bytes_consumed_total) - + rk->rk_telemetry.rk_historic_c.share_bytes_consumed_total; + double seconds = (now_ns - ts_last) / 1e9; + + if (seconds > 1.0) + bytes_consumed_rate.double_value = (double)delta / seconds; + else + bytes_consumed_rate.double_value = (double)delta; + + return bytes_consumed_rate; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_fetch_size_avg(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t avg_share_fetch_size; + brokers_avg(rk, rkb_avg_share_fetch_size, 1, avg_share_fetch_size); + return avg_share_fetch_size; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_fetch_size_max(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t max_share_fetch_size; + brokers_max(rk, rkb_avg_share_fetch_size, 1, max_share_fetch_size); + return max_share_fetch_size; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_fetch_total(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t share_fetch_total; + int64_t current = rd_atomic64_get(&rk->rk_telemetry.share_fetch_total); + + if (!rk->rk_telemetry.delta_temporality) + share_fetch_total.int_value = current; + else + share_fetch_total.int_value = + current - rk->rk_telemetry.rk_historic_c.share_fetch_total; + + return share_fetch_total; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_fetch_rate(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t share_fetch_rate; + rd_ts_t ts_last = rk->rk_telemetry.rk_historic_c.ts_last; + int64_t delta = rd_atomic64_get(&rk->rk_telemetry.share_fetch_total) - + rk->rk_telemetry.rk_historic_c.share_fetch_total; + double seconds = (now_ns - ts_last) / 1e9; + + /* For sub-second intervals (e.g. the first reading) report the raw + * delta instead of dividing, to avoid a tiny denominator massively + * overstating the rate. Same logic as the other rate metrics. */ + if (seconds > 1.0) + share_fetch_rate.double_value = (double)delta / seconds; + else + share_fetch_rate.double_value = (double)delta; + + return share_fetch_rate; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_acknowledgements_send_total(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t acknowledgements_send_total; + int64_t current = + rd_atomic64_get(&rk->rk_telemetry.acknowledgements_send_total); + + if (!rk->rk_telemetry.delta_temporality) + acknowledgements_send_total.int_value = current; + else + acknowledgements_send_total.int_value = + current - + rk->rk_telemetry.rk_historic_c.acknowledgements_send_total; + + return acknowledgements_send_total; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_acknowledgements_send_rate(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t acknowledgements_send_rate; + rd_ts_t ts_last = rk->rk_telemetry.rk_historic_c.ts_last; + int64_t delta = + rd_atomic64_get(&rk->rk_telemetry.acknowledgements_send_total) - + rk->rk_telemetry.rk_historic_c.acknowledgements_send_total; + double seconds = (now_ns - ts_last) / 1e9; + + /* For sub-second intervals (e.g. the first reading) report the raw + * delta instead of dividing, to avoid a tiny denominator massively + * overstating the rate. Same logic as the other rate metrics. */ + if (seconds > 1.0) + acknowledgements_send_rate.double_value = + (double)delta / seconds; + else + acknowledgements_send_rate.double_value = (double)delta; + + return acknowledgements_send_rate; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_heartbeat_total(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t heartbeat_total; + int64_t current = rd_atomic64_get(&rk->rk_telemetry.heartbeat_total); + + if (!rk->rk_telemetry.delta_temporality) + heartbeat_total.int_value = current; + else + heartbeat_total.int_value = + current - rk->rk_telemetry.rk_historic_c.heartbeat_total; + + return heartbeat_total; +} + +static rd_kafka_telemetry_metric_value_t +calculate_share_heartbeat_rate(rd_kafka_t *rk, + rd_kafka_broker_t *rkb_selected, + rd_ts_t now_ns) { + rd_kafka_telemetry_metric_value_t heartbeat_rate; + rd_ts_t ts_last = rk->rk_telemetry.rk_historic_c.ts_last; + int64_t delta = rd_atomic64_get(&rk->rk_telemetry.heartbeat_total) - + rk->rk_telemetry.rk_historic_c.heartbeat_total; + double seconds = (now_ns - ts_last) / 1e9; + + if (seconds > 1.0) + heartbeat_rate.double_value = (double)delta / seconds; + else + heartbeat_rate.double_value = (double)delta; + + return heartbeat_rate; +} + static void reset_historical_metrics(rd_kafka_t *rk, rd_ts_t now_ns) { rd_kafka_broker_t *rkb; @@ -320,6 +539,19 @@ static void reset_historical_metrics(rd_kafka_t *rk, rd_ts_t now_ns) { rkb->rkb_telemetry.rkb_historic_c.connects = rd_atomic32_get(&rkb->rkb_c.connects); } + + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rk->rk_telemetry.rk_historic_c.share_fetch_total = + rd_atomic64_get(&rk->rk_telemetry.share_fetch_total); + rk->rk_telemetry.rk_historic_c.share_bytes_consumed_total = + rd_atomic64_get( + &rk->rk_telemetry.share_bytes_consumed_total); + rk->rk_telemetry.rk_historic_c.acknowledgements_send_total = + rd_atomic64_get( + &rk->rk_telemetry.acknowledgements_send_total); + rk->rk_telemetry.rk_historic_c.heartbeat_total = + rd_atomic64_get(&rk->rk_telemetry.heartbeat_total); + } } static const rd_kafka_telemetry_metric_value_calculator_t @@ -377,6 +609,44 @@ static const rd_kafka_telemetry_metric_value_calculator_t &calculate_consumer_commit_latency_max, }; +static const rd_kafka_telemetry_metric_value_calculator_t SHARE_CONSUMER_METRIC_VALUE_CALCULATORS + [RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRIC__CNT] = { + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_POLL_IDLE_RATIO_AVG] = + &calculate_share_consumer_poll_idle_ratio_avg, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_AVG] = + &calculate_share_time_between_poll_avg, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_MAX] = + &calculate_share_time_between_poll_max, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_TOTAL] = + &calculate_share_fetch_total, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_RATE] = + &calculate_share_fetch_rate, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_AVG] = + &calculate_share_fetch_latency_avg, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_MAX] = + &calculate_share_fetch_latency_max, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_TOTAL] = + &calculate_share_acknowledgements_send_total, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_RATE] = + &calculate_share_acknowledgements_send_rate, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_AVG] = + &calculate_throttle_avg, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_MAX] = + &calculate_throttle_max, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_TOTAL] = + &calculate_share_heartbeat_total, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_RATE] = + &calculate_share_heartbeat_rate, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_AVG] = + &calculate_share_fetch_size_avg, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_MAX] = + &calculate_share_fetch_size_max, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_BYTES_CONSUMED_TOTAL] = + &calculate_share_bytes_consumed_total, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_BYTES_CONSUMED_RATE] = + &calculate_share_bytes_consumed_rate, +}; + static const char *get_client_rack(const rd_kafka_t *rk) { return rk->rk_conf.client_rack && RD_KAFKAP_STR_LEN(rk->rk_conf.client_rack) @@ -778,7 +1048,8 @@ rd_buf_t *rd_kafka_telemetry_encode_metrics(rd_kafka_t *rk) { rd_avg_rollover( &rkb->rkb_telemetry.rd_avg_rollover.rkb_avg_throttle, &rkb->rkb_telemetry.rd_avg_current.rkb_avg_throttle); - if (rk->rk_type == RD_KAFKA_CONSUMER) { + if (rk->rk_type == RD_KAFKA_CONSUMER && + !RD_KAFKA_IS_SHARE_CONSUMER(rk)) { rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_rollover .rkb_avg_fetch_latency); rd_avg_rollover(&rkb->rkb_telemetry.rd_avg_rollover @@ -793,26 +1064,60 @@ rd_buf_t *rd_kafka_telemetry_encode_metrics(rd_kafka_t *rk) { &rkb->rkb_telemetry.rd_avg_current .rkb_avg_produce_latency); } + + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_latency); + rd_avg_rollover(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_latency, + &rkb->rkb_telemetry.rd_avg_current + .rkb_avg_share_fetch_latency); + rd_avg_destroy(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_size); + rd_avg_rollover(&rkb->rkb_telemetry.rd_avg_rollover + .rkb_avg_share_fetch_size, + &rkb->rkb_telemetry.rd_avg_current + .rkb_avg_share_fetch_size); + } } if (rk->rk_type == RD_KAFKA_CONSUMER) { - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_poll_idle_ratio); - rd_avg_rollover( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_poll_idle_ratio, - &rk->rk_telemetry.rd_avg_current.rk_avg_poll_idle_ratio); - - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_rebalance_latency); - rd_avg_rollover( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_rebalance_latency, - &rk->rk_telemetry.rd_avg_current.rk_avg_rebalance_latency); - - rd_avg_destroy( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_commit_latency); - rd_avg_rollover( - &rk->rk_telemetry.rd_avg_rollover.rk_avg_commit_latency, - &rk->rk_telemetry.rd_avg_current.rk_avg_commit_latency); + /* TODO: factor out a helper that performs the destroy + + * rollover of a metric's rd_avg_rollover/rd_avg_current pair + * in one call, to reduce the repetition below. */ + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) { + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_poll_idle_ratio); + rd_avg_rollover(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_poll_idle_ratio, + &rk->rk_telemetry.rd_avg_current + .rk_avg_share_poll_idle_ratio); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_time_between_poll); + rd_avg_rollover(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_share_time_between_poll, + &rk->rk_telemetry.rd_avg_current + .rk_avg_share_time_between_poll); + } else { + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_poll_idle_ratio); + rd_avg_rollover(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_poll_idle_ratio, + &rk->rk_telemetry.rd_avg_current + .rk_avg_poll_idle_ratio); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_rebalance_latency); + rd_avg_rollover(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_rebalance_latency, + &rk->rk_telemetry.rd_avg_current + .rk_avg_rebalance_latency); + rd_avg_destroy(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_commit_latency); + rd_avg_rollover(&rk->rk_telemetry.rd_avg_rollover + .rk_avg_commit_latency, + &rk->rk_telemetry.rd_avg_current + .rk_avg_commit_latency); + } } int resource_attributes_count = @@ -883,12 +1188,19 @@ rd_buf_t *rd_kafka_telemetry_encode_metrics(rd_kafka_t *rk) { for (i = 0; i < metrics_to_encode_count; i++) { rd_kafka_telemetry_metric_value_calculator_t - metric_value_calculator = - (rk->rk_type == RD_KAFKA_PRODUCER) - ? PRODUCER_METRIC_VALUE_CALCULATORS - [metrics_to_encode[i]] - : CONSUMER_METRIC_VALUE_CALCULATORS - [metrics_to_encode[i]]; + metric_value_calculator; + if (RD_KAFKA_IS_SHARE_CONSUMER(rk)) + metric_value_calculator = + SHARE_CONSUMER_METRIC_VALUE_CALCULATORS + [metrics_to_encode[i]]; + else if (rk->rk_type == RD_KAFKA_PRODUCER) + metric_value_calculator = + PRODUCER_METRIC_VALUE_CALCULATORS + [metrics_to_encode[i]]; + else + metric_value_calculator = + CONSUMER_METRIC_VALUE_CALCULATORS + [metrics_to_encode[i]]; if (info[metrics_to_encode[i]].is_per_broker) { rd_kafka_broker_t *rkb; diff --git a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_encode.h b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_encode.h similarity index 61% rename from lib/librdkafka-2.10.1/src/rdkafka_telemetry_encode.h rename to lib/librdkafka-2.15.0/src/rdkafka_telemetry_encode.h index 75c6bc896b6..5d108506a1b 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_telemetry_encode.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_telemetry_encode.h @@ -36,14 +36,18 @@ #define RD_KAFKA_TELEMETRY_METRIC_NODE_ID_ATTRIBUTE "node.id" #define RD_KAFKA_TELEMETRY_METRIC_INFO(rk) \ - (rk->rk_type == RD_KAFKA_PRODUCER \ - ? RD_KAFKA_TELEMETRY_PRODUCER_METRICS_INFO \ - : RD_KAFKA_TELEMETRY_CONSUMER_METRICS_INFO) + (RD_KAFKA_IS_SHARE_CONSUMER(rk) \ + ? RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRICS_INFO \ + : (rk->rk_type == RD_KAFKA_PRODUCER \ + ? RD_KAFKA_TELEMETRY_PRODUCER_METRICS_INFO \ + : RD_KAFKA_TELEMETRY_CONSUMER_METRICS_INFO)) #define RD_KAFKA_TELEMETRY_METRIC_CNT(rk) \ - (rk->rk_type == RD_KAFKA_PRODUCER \ - ? RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT \ - : RD_KAFKA_TELEMETRY_CONSUMER_METRIC__CNT) + (RD_KAFKA_IS_SHARE_CONSUMER(rk) \ + ? RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRIC__CNT \ + : (rk->rk_type == RD_KAFKA_PRODUCER \ + ? RD_KAFKA_TELEMETRY_PRODUCER_METRIC__CNT \ + : RD_KAFKA_TELEMETRY_CONSUMER_METRIC__CNT)) typedef enum { @@ -82,6 +86,27 @@ typedef enum { RD_KAFKA_TELEMETRY_CONSUMER_METRIC__CNT } rd_kafka_telemetry_consumer_metric_name_t; +typedef enum { + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_POLL_IDLE_RATIO_AVG, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_AVG, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_MAX, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_RATE, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_AVG, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_MAX, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_RATE, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_AVG, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_MAX, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_RATE, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_AVG, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_MAX, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_BYTES_CONSUMED_TOTAL, + RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_BYTES_CONSUMED_RATE, + RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRIC__CNT +} rd_kafka_telemetry_share_consumer_metric_name_t; + typedef union { int64_t int_value; double double_value; @@ -296,6 +321,139 @@ static const rd_kafka_telemetry_metric_info_t RD_KAFKA_TELEMETRY_CONSUMER_METRIC .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, }; +static const rd_kafka_telemetry_metric_info_t RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRICS_INFO + [RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRIC__CNT] = { + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_POLL_IDLE_RATIO_AVG] = + {.name = "consumer.share.poll.idle.ratio.avg", + .description = "The average fraction of time the consumer's " + "poll() is idle as opposed to waiting for the " + "user code to process records.", + .unit = "1", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_AVG] = + {.name = "consumer.share.time.between.poll.avg", + .description = + "The average delay between invocations of poll() in " + "milliseconds.", + .unit = "ms", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_TIME_BETWEEN_POLL_MAX] = + {.name = "consumer.share.time.between.poll.max", + .description = "The max delay between invocations of poll() in " + "milliseconds.", + .unit = "ms", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_TOTAL] = + {.name = "consumer.share.fetch.manager.fetch.total", + .description = "The total number of fetch requests.", + .unit = "1", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_RATE] = + {.name = "consumer.share.fetch.manager.fetch.rate", + .description = "The number of fetch requests per second.", + .unit = "1", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_AVG] = + {.name = "consumer.share.fetch.manager.fetch.latency.avg", + .description = "The average time taken for a fetch request.", + .unit = "ms", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_LATENCY_MAX] = + {.name = "consumer.share.fetch.manager.fetch.latency.max", + .description = "The maximum time taken for any fetch request.", + .unit = "ms", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_TOTAL] = + {.name = "consumer.share.fetch.manager.acknowledgements.send." + "total", + .description = "The total number of record acknowledgements " + "sent.", + .unit = "1", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_ACKNOWLEDGEMENTS_SEND_RATE] = + {.name = "consumer.share.fetch.manager.acknowledgements.send." + "rate", + .description = "The average number of record acknowledgements " + "sent per second.", + .unit = "1", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_AVG] = + {.name = "consumer.share.fetch.manager.fetch.throttle.time.avg", + .description = "The average throttle time in ms.", + .unit = "ms", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_THROTTLE_TIME_MAX] = + {.name = "consumer.share.fetch.manager.fetch.throttle.time.max", + .description = "The maximum throttle time in ms.", + .unit = "ms", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_TOTAL] = + {.name = "consumer.share.coordinator.heartbeat.total", + .description = "The total number of heartbeats.", + .unit = "1", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_COORDINATOR_HEARTBEAT_RATE] = + {.name = "consumer.share.coordinator.heartbeat.rate", + .description = "The number of heartbeats per second.", + .unit = "1", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_AVG] = + {.name = "consumer.share.fetch.manager.fetch.size.avg", + .description = "The average number of bytes fetched per request.", + .unit = "bytes", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_FETCH_SIZE_MAX] = + {.name = "consumer.share.fetch.manager.fetch.size.max", + .description = "The maximum number of bytes fetched per request.", + .unit = "bytes", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_BYTES_CONSUMED_TOTAL] = + {.name = "consumer.share.fetch.manager.bytes.consumed.total", + .description = "The total number of bytes fetched.", + .unit = "bytes", + .is_int = rd_true, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_SUM}, + [RD_KAFKA_TELEMETRY_METRIC_SHARE_CONSUMER_BYTES_CONSUMED_RATE] = + {.name = "consumer.share.fetch.manager.bytes.consumed.rate", + .description = "The average number of bytes fetched per " + "second.", + .unit = "bytes", + .is_int = rd_false, + .is_per_broker = rd_false, + .type = RD_KAFKA_TELEMETRY_METRIC_TYPE_GAUGE}, +}; + rd_buf_t *rd_kafka_telemetry_encode_metrics(rd_kafka_t *rk); #endif /* _RDKAFKA_RDKAFKA_TELEMETRY_ENCODE_H */ diff --git a/lib/librdkafka-2.10.1/src/rdkafka_timer.c b/lib/librdkafka-2.15.0/src/rdkafka_timer.c similarity index 98% rename from lib/librdkafka-2.10.1/src/rdkafka_timer.c rename to lib/librdkafka-2.15.0/src/rdkafka_timer.c index b62343269dd..f10604167bd 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_timer.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_timer.c @@ -340,6 +340,8 @@ void rd_kafka_timers_run(rd_kafka_timers_t *rkts, int timeout_us) { while ((rtmr = TAILQ_FIRST(&rkts->rkts_timers)) && rtmr->rtmr_next <= now) { rd_bool_t oneshot; + void (*callback)(rd_kafka_timers_t *rkts, void *arg); + void *arg; rd_kafka_timer_unschedule(rkts, rtmr); @@ -352,9 +354,12 @@ void rd_kafka_timers_run(rd_kafka_timers_t *rkts, int timeout_us) { if ((oneshot = rtmr->rtmr_oneshot)) rtmr->rtmr_interval = 0; + callback = rtmr->rtmr_callback; + arg = rtmr->rtmr_arg; + rd_kafka_timers_unlock(rkts); - rtmr->rtmr_callback(rkts, rtmr->rtmr_arg); + callback(rkts, arg); rd_kafka_timers_lock(rkts); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_timer.h b/lib/librdkafka-2.15.0/src/rdkafka_timer.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_timer.h rename to lib/librdkafka-2.15.0/src/rdkafka_timer.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_topic.c b/lib/librdkafka-2.15.0/src/rdkafka_topic.c similarity index 88% rename from lib/librdkafka-2.10.1/src/rdkafka_topic.c rename to lib/librdkafka-2.15.0/src/rdkafka_topic.c index 56f63930a88..e357ebf4f86 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_topic.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_topic.c @@ -518,6 +518,158 @@ rd_kafka_topic_t *rd_kafka_topic_new0(rd_kafka_t *rk, } +/** + * @brief Create a topic instance with a topic_id known at construction. + * + * Used by share-consumer paths where the heartbeat-derived + * assignment carries a topic_id and the corresponding name was + * resolved (typically via metadata cache lookup) before this + * function is called. + * + * Returns the existing rkt if one with the same topic_id is + * already registered. The supplied name is ignored in that + * case; the existing rkt's name is immutable post-construction. + * + * Differences from rd_kafka_topic_new0 (the by-name twin): + * + * - rkt_topic_id is set at construction, avoiding the in-place + * mutation path in rd_kafka_topic_metadata_update that breaks + * share-session bookkeeping when a topic is recreated. + * + * - The unassigned (UA) toppar is NOT created. Share consumers + * do not produce, do not buffer messages under an UA + * partition, and never call rd_kafka_topic_assign_uas. + * Skipping UA also drops one rkt refcount slot, which means + * the rkt's lifetime is tied to its rkt_p[] / rkt_desp + * toppars (and any caller keeps). The last rkt reference + * must be released via one of: + * + * (a) set_notexists' partition_cnt_update(rkt, 0) + * destroying all rktps; or + * (b) rd_kafka_topic_partitions_remove(rkt) on consumer or + * rk destroy. + * + * Any code path that decrements an rkt reference on this rkt + * must respect this invariant — a premature destroy releases + * rktps that callers may still hold. + * + * - Producer-relevant fields in rkt_conf are NOT initialized: + * partitioner, sticky_partition_linger, compression config + * (codec/level), queuing strategy / msg-order comparator, + * random_partitioner flag. If a producer ever needs this + * construction path, mirror the relevant blocks from + * rd_kafka_topic_new0 (search for the partitioner setup near + * the top of that function) before using. + * + * - Conf parameter omitted; rkt_conf is populated from + * rk->rk_conf.topic_conf (or an empty topic_conf) and copied + * in by value. No finalize step; finalize covers + * producer-side validation that does not apply here. Share + * consumer does not consume topic-level conf today. + * + * - Cache seeding looks up the metadata cache by topic_id, not + * name. The seeded metadata update sees the rkt's topic_id + * already matches the cache entry's, so the in-place mutation + * branch in rd_kafka_topic_metadata_update is skipped. + * + * @param rk + * @param topic Topic name; must be non-NULL. Callers on the share-consumer + * path resolve the name via the metadata cache or the cgrp's + * current assignment before calling, so this is a + * precondition rather than a runtime check. + * @param topic_id Topic id; must be non-zero. + * @param do_lock Take rk wrlock internally if rd_true; caller must hold + * rk_wrlock if rd_false. + * + * @returns the existing or newly created rkt with rkt_refcnt bumped. + * NULL on invalid input. + * + * @locks_required rk_wrlock if do_lock is rd_false + * @locks_acquired rk_wrlock if do_lock is rd_true + */ +rd_kafka_topic_t *rd_kafka_topic_new_with_id(rd_kafka_t *rk, + const char *topic, + rd_kafka_Uuid_t topic_id, + int do_lock) { + rd_kafka_topic_t *rkt; + const struct rd_kafka_metadata_cache_entry *rkmce; + rd_kafka_topic_conf_t *conf; + + rd_dassert(!RD_KAFKA_UUID_IS_ZERO(topic_id)); + rd_dassert(topic != NULL); + if (RD_KAFKA_UUID_IS_ZERO(topic_id) || !topic) + return NULL; + + if (do_lock) + rd_kafka_wrlock(rk); + + rkt = rd_kafka_topic_find_by_topic_id(rk, topic_id); + if (rkt) { + if (do_lock) + rd_kafka_wrunlock(rk); + return rkt; + } + + if (rk->rk_conf.topic_conf) + conf = rd_kafka_topic_conf_dup(rk->rk_conf.topic_conf); + else + conf = rd_kafka_topic_conf_new(); + + rkt = rd_calloc(1, sizeof(*rkt)); + memcpy(rkt->rkt_magic, "IRKT", 4); + + rkt->rkt_topic = rd_kafkap_str_new(topic, -1); + rkt->rkt_topic_id = topic_id; + rkt->rkt_rk = rk; + + rkt->rkt_ts_create = rd_clock(); + rkt->rkt_ts_state = rkt->rkt_ts_create; + + rkt->rkt_conf = *conf; + rd_free(conf); /* explicitly not rd_kafka_topic_destroy() + * since we dont want to rd_free internal members, + * just the placeholder. The internal members + * were copied on the line above. */ + + rd_avg_init(&rkt->rkt_avg_batchsize, RD_AVG_GAUGE, 0, + rk->rk_conf.max_msg_size, 2, + rk->rk_conf.stats_interval_ms ? 1 : 0); + rd_avg_init(&rkt->rkt_avg_batchcnt, RD_AVG_GAUGE, 0, + rk->rk_conf.batch_num_messages, 2, + rk->rk_conf.stats_interval_ms ? 1 : 0); + + rd_kafka_dbg(rk, TOPIC, "TOPIC", "New local topic: %.*s (id %s)", + RD_KAFKAP_STR_PR(rkt->rkt_topic), + rd_kafka_Uuid_base64str(&topic_id)); + + rd_list_init(&rkt->rkt_desp, 16, NULL); + rd_interval_init(&rkt->rkt_desp_refresh_intvl); + TAILQ_INIT(&rkt->rkt_saved_partmsgids); + rd_refcnt_init(&rkt->rkt_refcnt, 0); + rd_refcnt_init(&rkt->rkt_app_refcnt, 0); + + rd_kafka_topic_keep(rkt); + + rwlock_init(&rkt->rkt_lock); + + TAILQ_INSERT_TAIL(&rk->rk_topics, rkt, rkt_link); + rk->rk_topic_cnt++; + + rkmce = rd_kafka_metadata_cache_find_by_id(rk, topic_id, 1 /*valid*/); + if (rkmce && !rkmce->rkmce_mtopic.err) { + rd_kafka_topic_metadata_update( + rkt, &rkmce->rkmce_mtopic, + &rkmce->rkmce_metadata_internal_topic, + rkmce->rkmce_ts_insert); + } + + if (do_lock) + rd_kafka_wrunlock(rk); + + return rkt; +} + + /** * @brief Create new app topic handle. @@ -721,7 +873,14 @@ static int rd_kafka_toppar_leader_update(rd_kafka_topic_t *rkt, rktp->rktp_leader_epoch = leader_epoch; + /* Share consumer never fetches from a follower (KIP-392), so + * rktp_broker == leader always holds on the share path. Force + * the flag to false here so the "not migrating away from + * preferred replica" branch is structurally unreachable on the + * share path, rather than depending on the runtime equality + * happening to hold. */ fetching_from_follower = + !RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk) && leader != NULL && rktp->rktp_broker != NULL && rktp->rktp_broker->rkb_source != RD_KAFKA_INTERNAL && rktp->rktp_broker != leader; @@ -742,6 +901,17 @@ static int rd_kafka_toppar_leader_update(rd_kafka_topic_t *rkt, if (rktp->rktp_leader_id != leader_id || rktp->rktp_leader != leader) { /* Update leader if it has changed */ + rd_kafka_dbg( + rktp->rktp_rkt->rkt_rk, TOPIC, "LEADER", + "%s [%" PRId32 + "]: applying leader-id update %" PRId32 + " -> %" PRId32 " (epoch %" PRId32 "%s)", + rktp->rktp_rkt->rkt_topic->str, + rktp->rktp_partition, rktp->rktp_leader_id, + leader_id, leader_epoch, + RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk) + ? ", share-consumer" + : ""); rktp->rktp_leader_id = leader_id; if (rktp->rktp_leader) rd_kafka_broker_destroy(rktp->rktp_leader); @@ -755,10 +925,16 @@ static int rd_kafka_toppar_leader_update(rd_kafka_topic_t *rkt, "leader updated"); } - if (need_epoch_validation) { + if (need_epoch_validation && + !RD_KAFKA_IS_SHARE_CONSUMER(rktp->rktp_rkt->rkt_rk)) { /* Set offset validation position, * depending it if should continue with current position or - * with next fetch start position. */ + * with next fetch start position. + * + * Skipped for share consumers: the offset-validation flow + * issues OffsetForLeaderEpoch / ListOffsets and pipes + * results into rktp_fetchq, which the share path doesn't + * drain. */ rd_kafka_toppar_set_offset_validation_position( rktp, rd_kafka_toppar_fetch_decide_next_fetch_start_pos(rktp)); @@ -940,16 +1116,18 @@ static int rd_kafka_topic_partition_cnt_update(rd_kafka_topic_t *rkt, if (unlikely(rkt->rkt_partition_cnt != 0 && !rd_kafka_terminating(rkt->rkt_rk))) rd_kafka_log(rk, LOG_NOTICE, "PARTCNT", - "Topic %s partition count changed " + "Topic %s (id %s) partition count changed " "from %" PRId32 " to %" PRId32, - rkt->rkt_topic->str, rkt->rkt_partition_cnt, - partition_cnt); + rkt->rkt_topic->str, + rd_kafka_Uuid_base64str(&rkt->rkt_topic_id), + rkt->rkt_partition_cnt, partition_cnt); else rd_kafka_dbg(rk, TOPIC, "PARTCNT", - "Topic %s partition count changed " + "Topic %s (id %s) partition count changed " "from %" PRId32 " to %" PRId32, - rkt->rkt_topic->str, rkt->rkt_partition_cnt, - partition_cnt); + rkt->rkt_topic->str, + rd_kafka_Uuid_base64str(&rkt->rkt_topic_id), + rkt->rkt_partition_cnt, partition_cnt); /* Create and assign new partition list */ @@ -1545,7 +1723,14 @@ int rd_kafka_topic_metadata_update2( rd_kafka_wrlock(rkb->rkb_rk); - if (likely(mdt->topic != NULL)) { + if (RD_KAFKA_IS_SHARE_CONSUMER(rkb->rkb_rk) && + !RD_KAFKA_UUID_IS_ZERO(mdit->topic_id)) { + /* Share consumer identifies topics by id; looking up + * by name risks matching a recreated topic's response + * to a stale rkt that shares its name. */ + rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, + mdit->topic_id); + } else if (likely(mdt->topic != NULL)) { rkt = rd_kafka_topic_find(rkb->rkb_rk, mdt->topic, 0 /*!lock*/); } else { rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, @@ -2129,6 +2314,42 @@ void rd_kafka_local_topics_to_list(rd_kafka_t *rk, } +/** + * @brief Populate \p topic_ids with the topic_ids (Uuid pointers) of all + * locally known topics that have a non-zero topic_id. + * + * Used by share-consumer periodic refresh: the broker's response + * to a by-id Metadata request tells the client which local rkts + * no longer correspond to a live cluster-side topic, driving + * cleanup of stale rkts left behind by topic recreation. + * + * @remark \p rk lock MUST NOT be held + * @remark Elements are rd_kafka_Uuid_t * allocated via rd_kafka_Uuid_copy + * and the caller must use rd_list_Uuid_destroy as the list's + * free callback. + */ +void rd_kafka_local_topic_ids_to_list(rd_kafka_t *rk, rd_list_t *topic_ids) { + rd_kafka_topic_t *rkt; + + rd_kafka_rdlock(rk); + rd_list_grow(topic_ids, rk->rk_topic_cnt); + /* TODO KIP-932: Consider skipping rkts in the non-existent state + * to avoid re-querying ids the broker has already told us are + * gone. Today the re-query is harmless (the resulting + * set_notexists is idempotent and the resulting error stays at + * debug-log level for the share consumer propagator), but it + * costs a few bytes per stale rkt per refresh. Keep in mind that + * including non-existent rkts also acts as a recovery path if a + * previously-absent id reappears in metadata. */ + TAILQ_FOREACH(rkt, &rk->rk_topics, rkt_link) { + if (!RD_KAFKA_UUID_IS_ZERO(rkt->rkt_topic_id)) + rd_list_add(topic_ids, + rd_kafka_Uuid_copy(&rkt->rkt_topic_id)); + } + rd_kafka_rdunlock(rk); +} + + /** * @brief Unit test helper to set a topic's state to EXISTS * with the given number of partitions. diff --git a/lib/librdkafka-2.10.1/src/rdkafka_topic.h b/lib/librdkafka-2.15.0/src/rdkafka_topic.h similarity index 90% rename from lib/librdkafka-2.10.1/src/rdkafka_topic.h rename to lib/librdkafka-2.15.0/src/rdkafka_topic.h index 7035b188b0f..c6d3aaeefd2 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_topic.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_topic.h @@ -109,6 +109,21 @@ typedef struct rd_kafka_partition_leader_epoch_s { int32_t leader_epoch; } rd_kafka_partition_leader_epoch_t; +/** + * @brief Return \p topic if non-NULL, else a printable literal + * placeholder. + * + * Use at %s format args where the source is a raw topic name + * that may be NULL — most commonly an `mdt->topic` from a + * Metadata response (compact_nullable_string can yield NULL + * when the broker doesn't know a topic_id we asked about). + */ +static RD_INLINE RD_UNUSED const char * +rd_kafka_topic_name_str_safe(const char *topic) { + return topic ? topic : "(null)"; +} + + /** * Finds and returns a topic based on its topic_id, or NULL if not found. * The 'rkt' refcount is increased by one and the caller must call @@ -193,6 +208,19 @@ struct rd_kafka_topic_s { #define rd_kafka_topic_wrunlock(rkt) rwlock_wrunlock(&(rkt)->rkt_lock) +/** + * @brief Return the name of \p rkt if available, else a printable + * literal placeholder. Guards \p rkt, \p rkt->rkt_topic, and + * \p rkt->rkt_topic->str — each layer can be NULL on degenerate + * flows that this helper is meant to absorb. + */ +static RD_INLINE RD_UNUSED const char * +rd_kafka_topic_name_safe(const rd_kafka_topic_t *rkt) { + if (!rkt || !rkt->rkt_topic || !rkt->rkt_topic->str) + return "(null)"; + return rkt->rkt_topic->str; +} + /** * @brief Increase refcount and return topic object. @@ -231,6 +259,11 @@ rd_kafka_topic_t *rd_kafka_topic_new0(rd_kafka_t *rk, int *existing, int do_lock); +rd_kafka_topic_t *rd_kafka_topic_new_with_id(rd_kafka_t *rk, + const char *topic, + rd_kafka_Uuid_t topic_id, + int do_lock); + rd_kafka_topic_t *rd_kafka_topic_find_fl(const char *func, int line, rd_kafka_t *rk, @@ -327,6 +360,8 @@ void rd_kafka_local_topics_to_list(rd_kafka_t *rk, rd_list_t *topics, int *cache_cntp); +void rd_kafka_local_topic_ids_to_list(rd_kafka_t *rk, rd_list_t *topic_ids); + void rd_ut_kafka_topic_set_topic_exists(rd_kafka_topic_t *rkt, int partition_cnt, int32_t leader_id); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_transport.c b/lib/librdkafka-2.15.0/src/rdkafka_transport.c similarity index 96% rename from lib/librdkafka-2.10.1/src/rdkafka_transport.c rename to lib/librdkafka-2.15.0/src/rdkafka_transport.c index 3407fc4bec6..b7c691d762e 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_transport.c +++ b/lib/librdkafka-2.15.0/src/rdkafka_transport.c @@ -281,16 +281,14 @@ static ssize_t rd_kafka_transport_socket_recvmsg(rd_kafka_transport_t *rktrans, else if (r == 0) { /* Receive 0 after POLLIN event means * connection closed. */ - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: connection closed by " - "peer"); - rd_snprintf(errstr, errstr_size, "Disconnected"); + rd_snprintf(errstr, errstr_size, + "Disconnected: connection closed by " + "peer: receive 0 after POLLIN"); return -1; } else if (r == -1 && rd_socket_errno == ECONNRESET) { - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: connection " - "reset by peer"); - rd_snprintf(errstr, errstr_size, "Disconnected"); + rd_snprintf(errstr, errstr_size, + "Disconnected: connection " + "reset by peer"); return -1; } else if (r == -1) { rd_snprintf(errstr, errstr_size, "%s", @@ -344,10 +342,9 @@ static ssize_t rd_kafka_transport_socket_recv0(rd_kafka_transport_t *rktrans, } else if (unlikely(r == 0)) { /* Receive 0 after POLLIN event means * connection closed. */ - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: connection closed by " - "peer"); - rd_snprintf(errstr, errstr_size, "Disconnected"); + rd_snprintf(errstr, errstr_size, + "Disconnected: connection closed by " + "peer: receive 0 after POLLIN"); return -1; } @@ -717,11 +714,10 @@ static void rd_kafka_transport_io_event(rd_kafka_transport_t *rktrans, if (r == 0 /* handshake still in progress */ && (events & POLLHUP)) { - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: during " - "SSL connection handshake"); rd_kafka_broker_conn_closed( - rkb, RD_KAFKA_RESP_ERR__TRANSPORT, "Disconnected"); + rkb, RD_KAFKA_RESP_ERR__TRANSPORT, + "Disconnected: during " + "SSL connection handshake"); return; } @@ -743,12 +739,10 @@ static void rd_kafka_transport_io_event(rd_kafka_transport_t *rktrans, } if (events & POLLHUP) { - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: hung up from peer in " - "state AUTH_LEGACY"); rd_kafka_broker_fail(rkb, LOG_ERR, RD_KAFKA_RESP_ERR__AUTHENTICATION, - "Disconnected"); + "Disconnected: hung up from " + "peer in state AUTH_LEGACY"); return; } @@ -771,11 +765,10 @@ static void rd_kafka_transport_io_event(rd_kafka_transport_t *rktrans, } if (events & POLLHUP) { - rd_rkb_dbg(rktrans->rktrans_rkb, BROKER, "SOCKET", - "Disconnected: connection closed by " - "peer"); rd_kafka_broker_conn_closed( - rkb, RD_KAFKA_RESP_ERR__TRANSPORT, "Disconnected"); + rkb, RD_KAFKA_RESP_ERR__TRANSPORT, + "Disconnected: connection closed by " + "peer: POLLHUP"); return; } diff --git a/lib/librdkafka-2.10.1/src/rdkafka_transport.h b/lib/librdkafka-2.15.0/src/rdkafka_transport.h similarity index 96% rename from lib/librdkafka-2.10.1/src/rdkafka_transport.h rename to lib/librdkafka-2.15.0/src/rdkafka_transport.h index c5f73163f94..d906b44211f 100644 --- a/lib/librdkafka-2.10.1/src/rdkafka_transport.h +++ b/lib/librdkafka-2.15.0/src/rdkafka_transport.h @@ -88,6 +88,11 @@ void rd_kafka_transport_set_blocked(rd_kafka_transport_t *rktrans, } while (0) #endif +static RD_INLINE RD_UNUSED rd_bool_t +rd_kafka_transport_error_disconnected(const char *errstr) { + return strncmp(errstr, "Disconnected", 12) == 0; +} + void rd_kafka_transport_init(void); diff --git a/lib/librdkafka-2.10.1/src/rdkafka_transport_int.h b/lib/librdkafka-2.15.0/src/rdkafka_transport_int.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_transport_int.h rename to lib/librdkafka-2.15.0/src/rdkafka_transport_int.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_txnmgr.c b/lib/librdkafka-2.15.0/src/rdkafka_txnmgr.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_txnmgr.c rename to lib/librdkafka-2.15.0/src/rdkafka_txnmgr.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_txnmgr.h b/lib/librdkafka-2.15.0/src/rdkafka_txnmgr.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_txnmgr.h rename to lib/librdkafka-2.15.0/src/rdkafka_txnmgr.h diff --git a/lib/librdkafka-2.10.1/src/rdkafka_zstd.c b/lib/librdkafka-2.15.0/src/rdkafka_zstd.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_zstd.c rename to lib/librdkafka-2.15.0/src/rdkafka_zstd.c diff --git a/lib/librdkafka-2.10.1/src/rdkafka_zstd.h b/lib/librdkafka-2.15.0/src/rdkafka_zstd.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdkafka_zstd.h rename to lib/librdkafka-2.15.0/src/rdkafka_zstd.h diff --git a/lib/librdkafka-2.10.1/src/rdlist.c b/lib/librdkafka-2.15.0/src/rdlist.c similarity index 97% rename from lib/librdkafka-2.10.1/src/rdlist.c rename to lib/librdkafka-2.15.0/src/rdlist.c index 65e3eb97e04..9c341833c92 100644 --- a/lib/librdkafka-2.10.1/src/rdlist.c +++ b/lib/librdkafka-2.15.0/src/rdlist.c @@ -270,6 +270,23 @@ void rd_list_sort(rd_list_t *rl, int (*cmp)(const void *, const void *)) { rl->rl_flags |= RD_LIST_F_SORTED; } +rd_bool_t rd_list_is_sorted(const rd_list_t *rl, + int (*cmp)(const void *, const void *)) { + int i; + + if (unlikely(rl->rl_elems == NULL || rl->rl_cnt < 2)) + return rd_true; + + if (rl->rl_flags & RD_LIST_F_SORTED) + return rd_true; + + for (i = 0; i < rl->rl_cnt - 1; i++) { + if (cmp(rl->rl_elems[i], rl->rl_elems[i + 1]) > 0) + return rd_false; + } + return rd_true; +} + static void rd_list_destroy_elems(rd_list_t *rl) { int i; diff --git a/lib/librdkafka-2.10.1/src/rdlist.h b/lib/librdkafka-2.15.0/src/rdlist.h similarity index 97% rename from lib/librdkafka-2.10.1/src/rdlist.h rename to lib/librdkafka-2.15.0/src/rdlist.h index 3a1316c3899..3403b230d1c 100644 --- a/lib/librdkafka-2.10.1/src/rdlist.h +++ b/lib/librdkafka-2.15.0/src/rdlist.h @@ -200,6 +200,16 @@ int rd_list_remove_multi_cmp(rd_list_t *rl, void rd_list_sort(rd_list_t *rl, int (*cmp)(const void *, const void *)); +/** + * @brief Check if list is sorted using comparator. + * + * @returns rd_true if the list is sorted in ascending order, rd_false + * otherwise. + */ +rd_bool_t rd_list_is_sorted(const rd_list_t *rl, + int (*cmp)(const void *, const void *)); + + /** * Empties the list and frees elements (if there is a free_cb). */ diff --git a/lib/librdkafka-2.10.1/src/rdlog.c b/lib/librdkafka-2.15.0/src/rdlog.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdlog.c rename to lib/librdkafka-2.15.0/src/rdlog.c diff --git a/lib/librdkafka-2.10.1/src/rdlog.h b/lib/librdkafka-2.15.0/src/rdlog.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdlog.h rename to lib/librdkafka-2.15.0/src/rdlog.h diff --git a/lib/librdkafka-2.10.1/src/rdmap.c b/lib/librdkafka-2.15.0/src/rdmap.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdmap.c rename to lib/librdkafka-2.15.0/src/rdmap.c diff --git a/lib/librdkafka-2.10.1/src/rdmap.h b/lib/librdkafka-2.15.0/src/rdmap.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdmap.h rename to lib/librdkafka-2.15.0/src/rdmap.h diff --git a/lib/librdkafka-2.10.1/src/rdmurmur2.c b/lib/librdkafka-2.15.0/src/rdmurmur2.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdmurmur2.c rename to lib/librdkafka-2.15.0/src/rdmurmur2.c diff --git a/lib/librdkafka-2.10.1/src/rdmurmur2.h b/lib/librdkafka-2.15.0/src/rdmurmur2.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdmurmur2.h rename to lib/librdkafka-2.15.0/src/rdmurmur2.h diff --git a/lib/librdkafka-2.10.1/src/rdports.c b/lib/librdkafka-2.15.0/src/rdports.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdports.c rename to lib/librdkafka-2.15.0/src/rdports.c diff --git a/lib/librdkafka-2.10.1/src/rdports.h b/lib/librdkafka-2.15.0/src/rdports.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdports.h rename to lib/librdkafka-2.15.0/src/rdports.h diff --git a/lib/librdkafka-2.10.1/src/rdposix.h b/lib/librdkafka-2.15.0/src/rdposix.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdposix.h rename to lib/librdkafka-2.15.0/src/rdposix.h diff --git a/lib/librdkafka-2.15.0/src/rdrand.c b/lib/librdkafka-2.15.0/src/rdrand.c new file mode 100644 index 00000000000..4f6139e0949 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdrand.c @@ -0,0 +1,151 @@ +/* + * librd - Rapid Development C library + * + * Copyright (c) 2012-2022, Magnus Edenhill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "rd.h" +#include "rdrand.h" +#include "rdtime.h" +#include "tinycthread.h" +#include "rdmurmur2.h" + +#ifdef HAVE_OSSL_SECURE_RAND_BYTES +#include +#endif +#if HAVE_GETENTROPY +/* getentropy() can be present in one of these two */ +#include +#include +#endif + +/* Initial seed with time+thread id */ +unsigned int rd_seed() { + unsigned int seed = 0, rand_bytes_seed = 0; + struct timeval tv; + + if (rd_rand_bytes((unsigned char *)&rand_bytes_seed, + sizeof(rand_bytes_seed))) + return rand_bytes_seed; + + rd_gettimeofday(&tv, NULL); + seed = (unsigned int)(tv.tv_usec); + seed ^= thrd_current_id(); + + /* Apply the murmur2 hash to distribute entropy to + * the whole seed. */ + seed = (unsigned int)rd_murmur2(&seed, sizeof(seed)); + return seed; +} + +static int rd_rand() { + int rand_num; +#if HAVE_RAND_R + static RD_TLS unsigned int seed = 0; + if (unlikely(seed == 0)) { + seed = rd_seed(); + } + rand_num = rand_r(&seed); +#else + rand_num = rand(); +#endif + return rand_num; +} + +#if HAVE_OSSL_SECURE_RAND_BYTES +static rd_bool_t rd_rand_bytes_by_ossl(unsigned char *buf, int num) { + int res = -1; + int retries = 0; + while ((res = RAND_priv_bytes(buf, num)) != 1) { + if (++retries == 5) + break; + + rd_usleep(1000, 0); /* wait for more entropy */ + } + return res == 1; +} +#endif + +#ifdef _WIN32 +static rd_bool_t rd_rand_bytes_by_rand_s(unsigned char *buf, int num) { + unsigned int rand, retries = 0; + while (num > 0) { + errno_t err; + int i; + while ((err = rand_s(&rand)) != 0) { + if (++retries == 5) + return rd_false; + rd_usleep(1000, 0); /* wait for more entropy */ + } + retries = 0; + i = sizeof(int); + while (i-- > 0 && num > 0) { + *buf++ = (unsigned char)(rand & 0xff); + rand >>= 8; + num--; + } + } + return rd_true; +} +#endif + +rd_bool_t rd_rand_bytes(unsigned char *buf, unsigned int num) { +#if HAVE_OSSL_SECURE_RAND_BYTES + if (rd_rand_bytes_by_ossl(buf, num)) + return rd_true; +#endif +#if HAVE_GETENTROPY + if (getentropy(buf, (size_t)num) == 0) + return rd_true; +#endif +#ifdef _WIN32 + if (rd_rand_bytes_by_rand_s(buf, num)) + return rd_true; +#endif + return rd_false; +} + +int rd_jitter(int low, int high) { + int rand_num = rd_rand(); + return (low + (rand_num % ((high - low) + 1))); +} + +void rd_array_shuffle(void *base, size_t nmemb, size_t entry_size) { + int i; + void *tmp = rd_alloca(entry_size); + + /* FIXME: Optimized version for word-sized entries. */ + + for (i = (int)nmemb - 1; i > 0; i--) { + int j = rd_jitter(0, i); + if (unlikely(i == j)) + continue; + + memcpy(tmp, (char *)base + (i * entry_size), entry_size); + memcpy((char *)base + (i * entry_size), + (char *)base + (j * entry_size), entry_size); + memcpy((char *)base + (j * entry_size), tmp, entry_size); + } +} diff --git a/lib/librdkafka-2.10.1/src/rdrand.h b/lib/librdkafka-2.15.0/src/rdrand.h similarity index 71% rename from lib/librdkafka-2.10.1/src/rdrand.h rename to lib/librdkafka-2.15.0/src/rdrand.h index f86fb83e791..1f5121af5bb 100644 --- a/lib/librdkafka-2.10.1/src/rdrand.h +++ b/lib/librdkafka-2.15.0/src/rdrand.h @@ -29,6 +29,35 @@ #ifndef _RDRAND_H_ #define _RDRAND_H_ +#if WITH_SSL +#include +#if OPENSSL_VERSION_NUMBER >= 0x10101000L +#define HAVE_OSSL_SECURE_RAND_BYTES 1 +#define HAVE_SECURE_RAND_BYTES 1 +#endif +#endif +#ifndef HAVE_OSSL_SECURE_RAND_BYTES +#if HAVE_GETENTROPY || defined(_WIN32) +#define HAVE_SECURE_RAND_BYTES 1 +#else +#define HAVE_SECURE_RAND_BYTES 0 +#endif +#endif + +/** + * Returns a (non-secure) random seed for a pseudo-random number generator. + */ +unsigned int rd_seed(); + +/** + * Fills \p buf with \p num cryptographically secure random bytes. + * + * @param buf Buffer to fill. + * @param num Number of bytes to generate. + * + * @return rd_true on success, rd_false on failure. + */ +rd_bool_t rd_rand_bytes(unsigned char *buf, unsigned int num); /** * Returns a random (using rand(3)) number between 'low'..'high' (inclusive). diff --git a/lib/librdkafka-2.10.1/src/rdregex.c b/lib/librdkafka-2.15.0/src/rdregex.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdregex.c rename to lib/librdkafka-2.15.0/src/rdregex.c diff --git a/lib/librdkafka-2.10.1/src/rdregex.h b/lib/librdkafka-2.15.0/src/rdregex.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdregex.h rename to lib/librdkafka-2.15.0/src/rdregex.h diff --git a/lib/librdkafka-2.10.1/src/rdsignal.h b/lib/librdkafka-2.15.0/src/rdsignal.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdsignal.h rename to lib/librdkafka-2.15.0/src/rdsignal.h diff --git a/lib/librdkafka-2.10.1/src/rdstring.c b/lib/librdkafka-2.15.0/src/rdstring.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdstring.c rename to lib/librdkafka-2.15.0/src/rdstring.c diff --git a/lib/librdkafka-2.10.1/src/rdstring.h b/lib/librdkafka-2.15.0/src/rdstring.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdstring.h rename to lib/librdkafka-2.15.0/src/rdstring.h diff --git a/lib/librdkafka-2.10.1/src/rdsysqueue.h b/lib/librdkafka-2.15.0/src/rdsysqueue.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdsysqueue.h rename to lib/librdkafka-2.15.0/src/rdsysqueue.h diff --git a/lib/librdkafka-2.10.1/src/rdtime.h b/lib/librdkafka-2.15.0/src/rdtime.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdtime.h rename to lib/librdkafka-2.15.0/src/rdtime.h diff --git a/lib/librdkafka-2.10.1/src/rdtypes.h b/lib/librdkafka-2.15.0/src/rdtypes.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdtypes.h rename to lib/librdkafka-2.15.0/src/rdtypes.h diff --git a/lib/librdkafka-2.10.1/src/rdunittest.c b/lib/librdkafka-2.15.0/src/rdunittest.c similarity index 95% rename from lib/librdkafka-2.10.1/src/rdunittest.c rename to lib/librdkafka-2.15.0/src/rdunittest.c index 45d47661f95..fe0e2d84b16 100644 --- a/lib/librdkafka-2.10.1/src/rdunittest.c +++ b/lib/librdkafka-2.15.0/src/rdunittest.c @@ -423,14 +423,23 @@ extern int unittest_scram(void); #endif extern int unittest_assignors(void); extern int unittest_map(void); +extern int unittest_fetcher_share_filter_forward(void); +extern int unittest_share_acknowledge(void); +extern int rd_kafka_unittest_msgset_errors(void); #if WITH_CURL extern int unittest_http(void); #endif #if WITH_OAUTHBEARER_OIDC extern int unittest_sasl_oauthbearer_oidc(void); +extern int unittest_sasl_oauthbearer_oidc_jwt_bearer(void); +extern int unittest_sasl_oauthbearer_oidc_assertion(void); #endif +extern int unittest_admin(void); extern int unittest_telemetry(void); extern int unittest_telemetry_decode(void); +#if WITH_SSL +extern int unittest_ssl(void); +#endif int rd_unittest(void) { int fails = 0; @@ -471,9 +480,22 @@ int rd_unittest(void) { #endif #if WITH_OAUTHBEARER_OIDC {"sasl_oauthbearer_oidc", unittest_sasl_oauthbearer_oidc}, + {"sasl_oauthbearer_oidc_jwt_bearer", + unittest_sasl_oauthbearer_oidc_jwt_bearer}, + {"sasl_oauthbearer_oidc_assertion", + unittest_sasl_oauthbearer_oidc_assertion}, #endif + {"admin", unittest_admin}, {"telemetry", unittest_telemetry}, {"telemetry_decode", unittest_telemetry_decode}, + {"fetcher_share_filter_forward", + unittest_fetcher_share_filter_forward}, + {"share_acknowledge", unittest_share_acknowledge}, + {"msgset_errors", rd_kafka_unittest_msgset_errors}, + {"feature", unittest_feature}, +#if WITH_SSL + {"ssl", unittest_ssl}, +#endif {NULL}}; int i; const char *match = rd_getenv("RD_UT_TEST", NULL); diff --git a/lib/librdkafka-2.10.1/src/rdunittest.h b/lib/librdkafka-2.15.0/src/rdunittest.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdunittest.h rename to lib/librdkafka-2.15.0/src/rdunittest.h diff --git a/lib/librdkafka-2.15.0/src/rdunittest_acknowledge.c b/lib/librdkafka-2.15.0/src/rdunittest_acknowledge.c new file mode 100644 index 00000000000..874a0b94d49 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdunittest_acknowledge.c @@ -0,0 +1,1177 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @brief Unit tests for Share Consumer acknowledge APIs + * + * Tests the following public APIs: + * 1. rd_kafka_share_acknowledge() - Acknowledge delivered record with ACCEPT + * 2. rd_kafka_share_acknowledge_type() - Acknowledge delivered record with type + * 3. rd_kafka_share_acknowledge_offset() - Acknowledge record by offset + */ + +#include "rd.h" +#include "rdunittest.h" +#include "rdkafka_int.h" +#include "rdkafka_partition.h" + +/* Test topic_id for unit tests (non-zero UUID) */ +static const rd_kafka_Uuid_t ut_topic_id_t1 = {.most_significant_bits = 1, + .least_significant_bits = 1}; +static const rd_kafka_Uuid_t ut_topic_id_t2 = {.most_significant_bits = 2, + .least_significant_bits = 2}; + +/** + * @brief Create a rd_kafka_share_t instance for testing using the proper + * share consumer API with explicit acknowledgement mode. + */ +static rd_kafka_share_t *ut_ack_create_share_consumer(void) { + rd_kafka_conf_t *conf = rd_kafka_conf_new(); + char errstr[128]; + + if (rd_kafka_conf_set(conf, "group.id", "ut-share-ack", errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return NULL; + } + + /* Enable explicit acknowledgement mode for testing acknowledge APIs */ + if (rd_kafka_conf_set(conf, "share.acknowledgement.mode", "explicit", + errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return NULL; + } + + rd_kafka_share_t *rkshare = + rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + /* conf is consumed by rd_kafka_share_consumer_new on success */ + + return rkshare; +} + +/** + * @brief Register a topic with the rd_kafka_t instance and set its topic_id. + * + * This is needed for rd_kafka_share_acknowledge_offset() to find the topic + * by name and resolve its topic_id. + */ +static rd_kafka_topic_t *ut_ack_register_topic(rd_kafka_t *rk, + const char *topic, + rd_kafka_Uuid_t topic_id) { + rd_kafka_topic_t *rkt = rd_kafka_topic_new(rk, topic, NULL); + if (!rkt) + return NULL; + + /* Set the topic_id on the registered topic */ + rkt->rkt_topic_id = topic_id; + + return rkt; +} + +/** + * @brief Clear all entries from the inflight acks map. + * + * Used between tests to reset state while keeping the same rkshare. + */ +static void ut_ack_clear_inflight_map(rd_kafka_share_t *rkshare) { + if (!rkshare) + return; + + /* RD_MAP_CLEAR will call the free functions for keys and values */ + RD_MAP_CLEAR(&rkshare->rkshare_inflight_acks); + + /* Reset unacked count */ + rkshare->rkshare_unacked_cnt = 0; +} + +/** + * @brief Add a partition with acquired offsets to the rkshare inflight map. + * + * Creates an entry with all offsets in ACQUIRED state (delivered records). + * Delivered records can be acknowledged via record-based APIs. + */ +static void ut_ack_add_partition(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t start_offset, + int64_t end_offset) { + rd_kafka_topic_partition_private_t *parpriv; + rd_kafka_share_ack_batches_t *batches = rd_calloc(1, sizeof(*batches)); + + batches->rktpar = rd_kafka_topic_partition_new(topic, partition); + parpriv = rd_kafka_topic_partition_private_new(); + batches->rktpar->_private = parpriv; + + batches->response_leader_id = 1; + + int64_t size = end_offset - start_offset + 1; + batches->response_acquired_offsets_count = (int32_t)size; + + rd_list_init(&batches->entries, 1, NULL); + + rd_kafka_share_ack_batch_entry_t *entry = rd_calloc(1, sizeof(*entry)); + entry->start_offset = start_offset; + entry->end_offset = end_offset; + entry->size = size; + entry->types_cnt = (int32_t)size; + entry->types = rd_calloc(size, sizeof(*entry->types)); + + /* Initialize all offsets to ACQUIRED */ + for (int64_t i = 0; i < size; i++) { + entry->types[i] = RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + } + + rd_list_add(&batches->entries, entry); + + /* Mark as sorted */ + batches->entries.rl_flags |= RD_LIST_F_SORTED; + + rd_kafka_topic_partition_t *key = + rd_kafka_topic_partition_new(topic, partition); + RD_MAP_SET(&rkshare->rkshare_inflight_acks, key, batches); + + /* Increment unacked count for all ACQUIRED offsets */ + rkshare->rkshare_unacked_cnt += size; +} + + +/** + * @brief Set a specific offset as a GAP record. + */ +static void ut_ack_set_gap(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t offset) { + rd_kafka_topic_partition_t *lookup_key = + rd_kafka_topic_partition_new(topic, partition); + + rd_kafka_share_ack_batches_t *batches = + RD_MAP_GET(&rkshare->rkshare_inflight_acks, lookup_key); + rd_kafka_topic_partition_destroy(lookup_key); + + if (!batches) + return; + + rd_kafka_share_ack_batch_entry_t *entry; + int i; + RD_LIST_FOREACH(entry, &batches->entries, i) { + if (offset >= entry->start_offset && + offset <= entry->end_offset) { + int64_t idx = offset - entry->start_offset; + entry->types[idx] = RD_KAFKA_SHARE_INTERNAL_ACK_GAP; + return; + } + } +} + +/** + * @brief Get the ack type for a specific offset in the inflight map. + */ +static rd_kafka_share_internal_acknowledgement_type +ut_ack_get_type(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition, + int64_t offset) { + rd_kafka_topic_partition_t *lookup_key = + rd_kafka_topic_partition_new(topic, partition); + + rd_kafka_share_ack_batches_t *batches = + RD_MAP_GET(&rkshare->rkshare_inflight_acks, lookup_key); + rd_kafka_topic_partition_destroy(lookup_key); + + if (!batches) + return -99; /* Invalid marker */ + + rd_kafka_share_ack_batch_entry_t *entry; + int i; + RD_LIST_FOREACH(entry, &batches->entries, i) { + if (offset >= entry->start_offset && + offset <= entry->end_offset) { + int64_t idx = offset - entry->start_offset; + return entry->types[idx]; + } + } + + return -99; /* Invalid marker */ +} + +/** + * @brief Create a mock rd_kafka_message_t for testing. + * + * Uses a properly registered rd_kafka_topic_t to avoid issues with + * rd_kafka_topic_name() when the acknowledge APIs are called. + */ +static rd_kafka_message_t *ut_ack_create_message(rd_kafka_topic_t *rkt, + int32_t partition, + int64_t offset) { + rd_kafka_message_t *rkmessage = rd_calloc(1, sizeof(*rkmessage)); + rkmessage->rkt = rkt; + rkmessage->partition = partition; + rkmessage->offset = offset; + return rkmessage; +} + +/** + * @brief Destroy a mock rd_kafka_message_t. + */ +static void ut_ack_destroy_message(rd_kafka_message_t *rkmessage) { + if (rkmessage) + rd_free(rkmessage); +} + +/** + * @brief Test rd_kafka_share_acknowledge() - Basic ACCEPT acknowledgement. + * + * Verifies that rd_kafka_share_acknowledge() correctly updates an offset from + * ACQUIRED to ACCEPT state, and that adjacent offsets remain unchanged. + */ +static int ut_case_acknowledge_accept(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + /* Add partition with offsets 0-9 in ACQUIRED state */ + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + + /* Create a mock message for offset 5 */ + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 5); + + /* Verify offset 5 is currently ACQUIRED */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "offset 5 should be ACQUIRED before acknowledge"); + + /* Call rd_kafka_share_acknowledge */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge(rkshare, msg); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "acknowledge failed: %s", rd_kafka_err2str(err)); + + /* Verify offset 5 is now ACCEPT */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 5 should be ACCEPT after acknowledge"); + + /* Verify other offsets are still ACQUIRED */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 4) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "offset 4 should still be ACQUIRED"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 6) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "offset 6 should still be ACQUIRED"); + + ut_ack_destroy_message(msg); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + +/** + * @brief Test rd_kafka_share_acknowledge_type() - Acknowledge with various + * types. + * + * Tests that rd_kafka_share_acknowledge_type() correctly updates the offset + * to the specified type (REJECT or RELEASE). + */ +static int ut_case_acknowledge_type_reject(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 3); + + /* Acknowledge with REJECT */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_type( + rkshare, msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "acknowledge_type REJECT failed: %s", + rd_kafka_err2str(err)); + + /* Verify type changed to REJECT */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 3) == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "offset 3 should be REJECT"); + + ut_ack_destroy_message(msg); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + +/** + * @brief Test rd_kafka_share_acknowledge_type() with RELEASE type. + */ +static int ut_case_acknowledge_type_release(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 7); + + /* Acknowledge with RELEASE */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_type( + rkshare, msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "acknowledge_type RELEASE failed: %s", + rd_kafka_err2str(err)); + + /* Verify type changed to RELEASE */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 7) == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "offset 7 should be RELEASE"); + + ut_ack_destroy_message(msg); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + +/** + * @brief Test re-acknowledgement with record-based APIs. + * + * Verifies that re-acknowledging a delivered record with record-based APIs + * succeeds and updates the type. This is allowed before sending the + * acknowledgement to the broker. + */ +static int ut_case_reacknowledge_delivered(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 5); + + /* First acknowledge with ACCEPT */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_type( + rkshare, msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "first acknowledge failed: %s", rd_kafka_err2str(err)); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 5 should be ACCEPT"); + + /* Re-acknowledge with REJECT - should succeed */ + err = rd_kafka_share_acknowledge_type( + rkshare, msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "re-acknowledge should succeed, got %s", + rd_kafka_err2str(err)); + + /* Verify type changed to REJECT */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "offset 5 should be REJECT after re-acknowledge"); + + /* Re-acknowledge again with RELEASE */ + err = rd_kafka_share_acknowledge_type( + rkshare, msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "second re-acknowledge should succeed"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "offset 5 should be RELEASE after second re-acknowledge"); + + ut_ack_destroy_message(msg); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + +/** + * @brief Test error case - GAP records cannot be acknowledged. + * + * Verifies that GAP records cannot be acknowledged by any API. + */ +static int ut_case_error_gap_record(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + + /* Set offset 5 as a GAP record */ + ut_ack_set_gap(rkshare, topic, 0, 5); + + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 5); + + /* Try to acknowledge GAP with record-based API - should fail */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge(rkshare, msg); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for GAP record, got %s", + rd_kafka_err2str(err)); + + /* Verify GAP record is unchanged */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_GAP, + "offset 5 should still be GAP"); + + ut_ack_destroy_message(msg); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + +/** + * @brief Test error case - Invalid parameters (NULL). + * + * Verifies that a NULL rkshare returns RD_KAFKA_RESP_ERR__STATE (rejected + * by the reentrancy guard), and that NULL message or topic parameters + * return RD_KAFKA_RESP_ERR__INVALID_ARG. + */ +static int ut_case_error_null_parameters(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 5); + + /* Test NULL rkshare */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge(NULL, msg); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE for NULL rkshare, got %s", + rd_kafka_err2str(err)); + + /* Test NULL message */ + err = rd_kafka_share_acknowledge(rkshare, NULL); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "expected INVALID_ARG for NULL message, got %s", + rd_kafka_err2str(err)); + + /* Test NULL topic in acknowledge_offset */ + err = rd_kafka_share_acknowledge_offset( + rkshare, NULL, 0, 5, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "expected INVALID_ARG for NULL topic, got %s", + rd_kafka_err2str(err)); + + ut_ack_destroy_message(msg); + + RD_UT_PASS(); +} + +/** + * @brief Test error case - Invalid acknowledgement type. + * + * Verifies that invalid acknowledgement types (e.g., GAP which is + * internal-only, or arbitrary values like 99) return + * RD_KAFKA_RESP_ERR__INVALID_ARG error. Also verifies the offset remains in + * ACQUIRED state after the failed attempt. + */ +static int ut_case_error_invalid_type(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + + rd_kafka_message_t *msg = ut_ack_create_message(rkt, 0, 5); + + /* Test invalid type value (e.g., 99) */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_type( + rkshare, msg, (rd_kafka_share_AcknowledgeType_t)99); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "expected INVALID_ARG for invalid type, got %s", + rd_kafka_err2str(err)); + + /* Test type 0 (GAP - not allowed in public API) */ + err = rd_kafka_share_acknowledge_type( + rkshare, msg, (rd_kafka_share_AcknowledgeType_t)0); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "expected INVALID_ARG for GAP type, got %s", + rd_kafka_err2str(err)); + + /* Verify offset is still ACQUIRED (not modified) */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "offset 5 should still be ACQUIRED after invalid type"); + + ut_ack_destroy_message(msg); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + +/** + * @brief Test multiple partitions with record-based APIs. + * + * Tests acknowledging delivered records across multiple topics and partitions + * using record-based APIs. Verifies that each partition's acknowledgements + * are tracked independently. + */ +static int ut_case_acknowledge_multiple_partitions(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt_t1, + rd_kafka_topic_t *rkt_t2) { + const char *topic1 = rd_kafka_topic_name(rkt_t1); + const char *topic2 = rd_kafka_topic_name(rkt_t2); + + /* Add multiple partitions with delivered records */ + ut_ack_add_partition(rkshare, topic1, 0, 0, 9); + ut_ack_add_partition(rkshare, topic1, 1, 100, 109); + ut_ack_add_partition(rkshare, topic2, 0, 50, 59); + + rd_kafka_message_t *msg1 = ut_ack_create_message(rkt_t1, 0, 5); + rd_kafka_message_t *msg2 = ut_ack_create_message(rkt_t1, 1, 105); + rd_kafka_message_t *msg3 = ut_ack_create_message(rkt_t2, 0, 55); + + rd_kafka_resp_err_t err; + + /* Acknowledge across partitions using record-based APIs */ + err = rd_kafka_share_acknowledge_type( + rkshare, msg1, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "T1-0 offset 5 failed"); + + err = rd_kafka_share_acknowledge_type( + rkshare, msg2, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "T1-1 offset 105 failed"); + + err = rd_kafka_share_acknowledge_type( + rkshare, msg3, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "T2-0 offset 55 failed"); + + /* Verify each partition independently */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic1, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "T1-0 offset 5 should be ACCEPT"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic1, 1, 105) == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "T1-1 offset 105 should be REJECT"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic2, 0, 55) == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "T2-0 offset 55 should be RELEASE"); + + /* Verify other offsets unchanged */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic1, 0, 4) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "T1-0 offset 4 should be ACQUIRED"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic1, 1, 104) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "T1-1 offset 104 should be ACQUIRED"); + + ut_ack_destroy_message(msg1); + ut_ack_destroy_message(msg2); + ut_ack_destroy_message(msg3); + ut_ack_clear_inflight_map(rkshare); + + RD_UT_PASS(); +} + + +/** + * @brief Add a partition with multiple non-contiguous entries for binary search + * tests. + * + * Creates entries at: [0-9], [100-109], [200-209], [500-509] + * This tests binary search with gaps between entries. + */ +static void ut_ack_add_partition_multiple_entries(rd_kafka_share_t *rkshare, + const char *topic, + int32_t partition) { + rd_kafka_topic_partition_private_t *parpriv; + rd_kafka_share_ack_batches_t *batches = rd_calloc(1, sizeof(*batches)); + + batches->rktpar = rd_kafka_topic_partition_new(topic, partition); + parpriv = rd_kafka_topic_partition_private_new(); + batches->rktpar->_private = parpriv; + + batches->response_leader_id = 1; + batches->response_acquired_offsets_count = 40; + + rd_list_init(&batches->entries, 4, NULL); + + /* Entry 1: offsets 0-9 */ + rd_kafka_share_ack_batch_entry_t *entry1 = + rd_calloc(1, sizeof(*entry1)); + entry1->start_offset = 0; + entry1->end_offset = 9; + entry1->size = 10; + entry1->types_cnt = 10; + entry1->types = rd_calloc(10, sizeof(*entry1->types)); + for (int i = 0; i < 10; i++) + entry1->types[i] = RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + rd_list_add(&batches->entries, entry1); + + /* Entry 2: offsets 100-109 */ + rd_kafka_share_ack_batch_entry_t *entry2 = + rd_calloc(1, sizeof(*entry2)); + entry2->start_offset = 100; + entry2->end_offset = 109; + entry2->size = 10; + entry2->types_cnt = 10; + entry2->types = rd_calloc(10, sizeof(*entry2->types)); + for (int i = 0; i < 10; i++) + entry2->types[i] = RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + rd_list_add(&batches->entries, entry2); + + /* Entry 3: offsets 200-209 */ + rd_kafka_share_ack_batch_entry_t *entry3 = + rd_calloc(1, sizeof(*entry3)); + entry3->start_offset = 200; + entry3->end_offset = 209; + entry3->size = 10; + entry3->types_cnt = 10; + entry3->types = rd_calloc(10, sizeof(*entry3->types)); + for (int i = 0; i < 10; i++) + entry3->types[i] = RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + rd_list_add(&batches->entries, entry3); + + /* Entry 4: offsets 500-509 */ + rd_kafka_share_ack_batch_entry_t *entry4 = + rd_calloc(1, sizeof(*entry4)); + entry4->start_offset = 500; + entry4->end_offset = 509; + entry4->size = 10; + entry4->types_cnt = 10; + entry4->types = rd_calloc(10, sizeof(*entry4->types)); + for (int i = 0; i < 10; i++) + entry4->types[i] = RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED; + rd_list_add(&batches->entries, entry4); + + /* Mark as sorted for binary search */ + batches->entries.rl_flags |= RD_LIST_F_SORTED; + + rd_kafka_topic_partition_t *key = + rd_kafka_topic_partition_new(topic, partition); + RD_MAP_SET(&rkshare->rkshare_inflight_acks, key, batches); + + /* Increment unacked count for all ACQUIRED offsets (4 entries × 10) */ + rkshare->rkshare_unacked_cnt += 40; +} + +/** + * @brief Test binary search - find offset in first entry. + * + * Tests that rd_list_find correctly finds offsets at the beginning of the list. + */ +static int ut_case_bsearch_first_entry(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition_multiple_entries(rkshare, topic, 0); + + /* Acknowledge offset 5 in first entry [0-9] */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 5, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "first entry offset 5 failed: %s", rd_kafka_err2str(err)); + + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 5) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 5 should be ACCEPT"); + + /* Also test first offset of first entry */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "first entry offset 0 failed: %s", rd_kafka_err2str(err)); + + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 0) == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "offset 0 should be REJECT"); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - find offset in last entry. + * + * Tests that rd_list_find correctly finds offsets at the end of the list. + */ +static int ut_case_bsearch_last_entry(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition_multiple_entries(rkshare, topic, 0); + + /* Acknowledge offset 505 in last entry [500-509] */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 505, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "last entry offset 505 failed: %s", rd_kafka_err2str(err)); + + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 505) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 505 should be ACCEPT"); + + /* Also test last offset of last entry */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 509, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "last entry offset 509 failed: %s", rd_kafka_err2str(err)); + + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 509) == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "offset 509 should be RELEASE"); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - find offset in middle entries. + * + * Tests that rd_list_find correctly finds offsets in middle entries. + */ +static int ut_case_bsearch_middle_entries(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition_multiple_entries(rkshare, topic, 0); + + /* Acknowledge in second entry [100-109] */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 105, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "second entry offset 105 failed: %s", + rd_kafka_err2str(err)); + + /* Acknowledge in third entry [200-209] */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 200, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "third entry offset 200 failed: %s", + rd_kafka_err2str(err)); + + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 105) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 105 should be ACCEPT"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 200) == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "offset 200 should be REJECT"); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - offset not found (before all entries). + * + * Tests that rd_list_find returns NULL for offsets before the first entry. + * This is a special case where offset < first_entry->start_offset. + * + * Note: This test requires a setup where the first entry doesn't start at 0. + */ +static int ut_case_bsearch_not_found_before(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + /* Add entries starting at offset 100 */ + ut_ack_add_partition(rkshare, topic, 0, 100, 109); + + /* Try to acknowledge offset 50 (before all entries) */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 50, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for offset before entries, got %s", + rd_kafka_err2str(err)); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - offset not found (after all entries). + * + * Tests that rd_list_find returns NULL for offsets after the last entry. + */ +static int ut_case_bsearch_not_found_after(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition_multiple_entries(rkshare, topic, 0); + + /* Try to acknowledge offset 600 (after last entry [500-509]) */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 600, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for offset after entries, got %s", + rd_kafka_err2str(err)); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - offset not found (in gap between entries). + * + * Tests that rd_list_find returns NULL for offsets in gaps between entries. + * Entries: [0-9], [100-109], [200-209], [500-509] + * Gaps: 10-99, 110-199, 210-499 + */ +static int ut_case_bsearch_not_found_gap(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition_multiple_entries(rkshare, topic, 0); + + /* Try to acknowledge offset 50 (in gap between entry 1 and 2) */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 50, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for offset 50 in gap, got %s", + rd_kafka_err2str(err)); + + /* Try to acknowledge offset 150 (in gap between entry 2 and 3) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 150, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for offset 150 in gap, got %s", + rd_kafka_err2str(err)); + + /* Try to acknowledge offset 300 (in gap between entry 3 and 4) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 300, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for offset 300 in gap, got %s", + rd_kafka_err2str(err)); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - empty entries list. + * + * Tests that rd_list_find handles an empty entries list gracefully. + */ +static int ut_case_bsearch_empty_entries(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + /* Create a batches entry with empty entries list */ + rd_kafka_topic_partition_private_t *parpriv; + rd_kafka_share_ack_batches_t *batches = rd_calloc(1, sizeof(*batches)); + + batches->rktpar = rd_kafka_topic_partition_new(topic, 0); + parpriv = rd_kafka_topic_partition_private_new(); + batches->rktpar->_private = parpriv; + batches->response_leader_id = 1; + batches->response_acquired_offsets_count = 0; + + rd_list_init(&batches->entries, 0, NULL); + batches->entries.rl_flags |= RD_LIST_F_SORTED; + + rd_kafka_topic_partition_t *key = + rd_kafka_topic_partition_new(topic, 0); + RD_MAP_SET(&rkshare->rkshare_inflight_acks, key, batches); + + /* Try to acknowledge any offset */ + rd_kafka_resp_err_t err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 100, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "expected STATE error for empty entries, got %s", + rd_kafka_err2str(err)); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test binary search - boundary offsets of entries. + * + * Tests that rd_list_find correctly handles the exact start and end offsets + * of each entry (boundary conditions). + */ +static int ut_case_bsearch_boundary_offsets(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + + ut_ack_add_partition_multiple_entries(rkshare, topic, 0); + + rd_kafka_resp_err_t err; + + /* Test boundary offsets for each entry */ + /* Entry 1: [0-9] - test 0 (start) and 9 (end) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 0 failed"); + + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 9, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 9 failed"); + + /* Entry 2: [100-109] - test 100 (start) and 109 (end) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 100, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 100 failed"); + + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 109, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 109 failed"); + + /* Entry 3: [200-209] - test 200 (start) and 209 (end) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 200, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 200 failed"); + + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 209, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 209 failed"); + + /* Entry 4: [500-509] - test 500 (start) and 509 (end) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 500, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 500 failed"); + + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 509, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "offset 509 failed"); + + /* Verify all boundary offsets are ACCEPT */ + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 0) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 0 type"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 9) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 9 type"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 100) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 100 type"); + RD_UT_ASSERT(ut_ack_get_type(rkshare, topic, 0, 109) == + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "offset 109 type"); + + /* Test that offset 10 (just after entry 1) is not found */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 10, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "offset 10 should not be found, got %s", + rd_kafka_err2str(err)); + + /* Test that offset 99 (just before entry 2) is not found */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 99, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "offset 99 should not be found, got %s", + rd_kafka_err2str(err)); + + ut_ack_clear_inflight_map(rkshare); + RD_UT_PASS(); +} + +/** + * @brief Test unacked count tracking. + * + * Verifies that rkshare_unacked_cnt is properly: + * - Incremented when ACQUIRED records are added + * - Decremented when records are acknowledged + * - Not decremented for GAP records or re-acknowledgements + */ +static int ut_case_unacked_count(rd_kafka_share_t *rkshare, + rd_kafka_topic_t *rkt) { + const char *topic = rd_kafka_topic_name(rkt); + rd_kafka_resp_err_t err; + + /* Start with clean state */ + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "initial unacked count should be 0, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + /* Add partition with 10 offsets (0-9) - all ACQUIRED */ + ut_ack_add_partition(rkshare, topic, 0, 0, 9); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 10, + "unacked count after adding 10 offsets should be 10, " + "got %" PRId64, + rkshare->rkshare_unacked_cnt); + + /* Acknowledge offset 5 - should decrement count */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 5, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "acknowledge offset 5 failed: %s", rd_kafka_err2str(err)); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 9, + "unacked count after acking offset 5 should be 9, " + "got %" PRId64, + rkshare->rkshare_unacked_cnt); + + /* Re-acknowledge same offset - should NOT decrement (already acked) */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 5, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "re-acknowledge offset 5 failed: %s", + rd_kafka_err2str(err)); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 9, + "unacked count after re-acking should still be 9, " + "got %" PRId64, + rkshare->rkshare_unacked_cnt); + + /* Set offset 3 as GAP */ + ut_ack_set_gap(rkshare, topic, 0, 3); + + /* Try to acknowledge GAP - should fail and NOT change count */ + int64_t count_before = rkshare->rkshare_unacked_cnt; + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 3, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "GAP acknowledge should fail"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == count_before, + "unacked count should not change for GAP, " + "expected %" PRId64 " got %" PRId64, + count_before, rkshare->rkshare_unacked_cnt); + + /* Acknowledge multiple offsets */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "ack offset 0 failed"); + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 1, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "ack offset 1 failed"); + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 2, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + RD_UT_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, "ack offset 2 failed"); + + /* 9 - 3 = 6 remaining (offsets 4,6,7,8,9 are ACQUIRED, 3 is GAP) */ + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 6, + "unacked count after acking 0,1,2 should be 6, " + "got %" PRId64, + rkshare->rkshare_unacked_cnt); + + ut_ack_clear_inflight_map(rkshare); + + /* After clear, count should be 0 */ + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "unacked count after clear should be 0, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + RD_UT_PASS(); +} + + +/** + * @brief Main entry point for Share Consumer acknowledge API unit tests. + */ +int unittest_share_acknowledge(void) { + rd_kafka_share_t *rkshare; + rd_kafka_t *rk; + rd_kafka_topic_t *rkt_t1, *rkt_t2; + + RD_UT_SAY("==============================================="); + RD_UT_SAY("Share Consumer Acknowledge API Unit Tests"); + RD_UT_SAY("==============================================="); + + rkshare = ut_ack_create_share_consumer(); + RD_UT_ASSERT(rkshare != NULL, "Failed to create rd_kafka_share_t"); + + rk = rkshare->rkshare_rk; + + /* Register topics T1 and T2 upfront for all tests that need them. + * These will be destroyed at the end of the test suite. */ + rkt_t1 = ut_ack_register_topic(rk, "T1", ut_topic_id_t1); + RD_UT_ASSERT(rkt_t1 != NULL, "Failed to register topic T1"); + rkt_t2 = ut_ack_register_topic(rk, "T2", ut_topic_id_t2); + RD_UT_ASSERT(rkt_t2 != NULL, "Failed to register topic T2"); + +/* Macro for test cleanup on failure */ +#define UT_ACK_CLEANUP_AND_FAIL() \ + do { \ + rd_kafka_topic_destroy(rkt_t1); \ + rd_kafka_topic_destroy(rkt_t2); \ + rd_kafka_share_consumer_close(rkshare); \ + rd_kafka_share_destroy(rkshare); \ + return 1; \ + } while (0) + + /* Record-based API tests (delivered records) */ + RD_UT_SAY("Testing rd_kafka_share_acknowledge() (ACCEPT)..."); + if (ut_case_acknowledge_accept(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing rd_kafka_share_acknowledge_type() (REJECT)..."); + if (ut_case_acknowledge_type_reject(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing rd_kafka_share_acknowledge_type() (RELEASE)..."); + if (ut_case_acknowledge_type_release(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + /* Re-acknowledgement tests */ + RD_UT_SAY("Testing re-acknowledgement of delivered records..."); + if (ut_case_reacknowledge_delivered(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + /* Error case tests */ + RD_UT_SAY("Testing error: GAP records cannot be acknowledged..."); + if (ut_case_error_gap_record(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing error: NULL parameters..."); + if (ut_case_error_null_parameters(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing error: invalid type..."); + if (ut_case_error_invalid_type(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + /* Multi-partition test */ + RD_UT_SAY("Testing multiple partitions..."); + if (ut_case_acknowledge_multiple_partitions(rkshare, rkt_t1, rkt_t2)) + UT_ACK_CLEANUP_AND_FAIL(); + + /* Binary search tests (multiple entries) */ + RD_UT_SAY("Testing binary search: first entry..."); + if (ut_case_bsearch_first_entry(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: last entry..."); + if (ut_case_bsearch_last_entry(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: middle entries..."); + if (ut_case_bsearch_middle_entries(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: offset before all entries..."); + if (ut_case_bsearch_not_found_before(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: offset after all entries..."); + if (ut_case_bsearch_not_found_after(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: offset in gap between entries..."); + if (ut_case_bsearch_not_found_gap(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: empty entries list..."); + if (ut_case_bsearch_empty_entries(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + RD_UT_SAY("Testing binary search: boundary offsets..."); + if (ut_case_bsearch_boundary_offsets(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + + /* Unacked count tracking test */ + RD_UT_SAY("Testing unacked count tracking..."); + if (ut_case_unacked_count(rkshare, rkt_t1)) + UT_ACK_CLEANUP_AND_FAIL(); + +#undef UT_ACK_CLEANUP_AND_FAIL + + rd_kafka_topic_destroy(rkt_t1); + rd_kafka_topic_destroy(rkt_t2); + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); + RD_UT_PASS(); +} diff --git a/lib/librdkafka-2.15.0/src/rdunittest_fetcher.c b/lib/librdkafka-2.15.0/src/rdunittest_fetcher.c new file mode 100644 index 00000000000..29c1a9a1026 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdunittest_fetcher.c @@ -0,0 +1,1211 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @name Unit tests for Share Consumer acknowledgement flow + * @{ + * + * Tests cover: + * - rd_kafka_share_filter_acquired_records_and_update_ack_type() + * - rd_kafka_share_build_ack_details() + * - rd_kafka_share_build_inflight_acks_map() + */ + +#include "rd.h" +#include "rdunittest.h" +#include "rdkafka_int.h" +#include "rdkafka_queue.h" +#include "rdkafka_fetcher.h" +#include "rdkafka_partition.h" +#include "rdkafka_share_acknowledgement.h" + + +/** + * @name Test helpers + * @{ + */ + +static rd_kafka_t *ut_mock_rk; + +static rd_kafka_share_t *ut_create_mock_rkshare(void) { + rd_kafka_conf_t *conf = rd_kafka_conf_new(); + char errstr[128]; + + if (rd_kafka_conf_set(conf, "group.id", "ut-share-ack", errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return NULL; + } + + rd_kafka_share_t *rkshare = + rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + + return rkshare; +} + +static void ut_destroy_mock_rkshare(rd_kafka_share_t *rkshare) { + if (!rkshare) + return; + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); +} + +static rd_kafka_topic_partition_t * +ut_create_rktpar_with_id(const char *topic, + int32_t partition, + rd_kafka_Uuid_t topic_id) { + return rd_kafka_topic_partition_new_with_id_and_name(topic_id, topic, + partition); +} + +static void ut_add_batches_to_map(rd_kafka_share_t *rkshare, + rd_kafka_share_ack_batches_t *batches) { + rd_kafka_topic_partition_t *key = rd_kafka_topic_partition_new( + batches->rktpar->topic, batches->rktpar->partition); + /* Mark entries as sorted */ + batches->entries.rl_flags |= RD_LIST_F_SORTED; + RD_MAP_SET(&rkshare->rkshare_inflight_acks, key, batches); +} + +static rd_kafka_q_t *ut_create_mock_queue(void) { + rd_kafka_q_t *rkq = rd_calloc(1, sizeof(*rkq)); + rd_kafka_q_init(rkq, NULL); + rkq->rkq_flags |= RD_KAFKA_Q_F_ALLOCATED; + return rkq; +} + +static void ut_destroy_mock_queue(rd_kafka_q_t *rkq) { + if (!rkq) + return; + rd_kafka_q_destroy_owner(rkq); +} + +static rd_kafka_op_t *ut_create_fetch_op_with_offset(int64_t offset) { + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_FETCH); + rko->rko_u.fetch.rkm.rkm_rkmessage.offset = offset; + return rko; +} + +static void ut_op_destroy_free(void *ptr) { + rd_kafka_op_destroy((rd_kafka_op_t *)ptr); +} + +static rd_kafka_op_t *ut_create_share_fetch_response_rko(void) { + rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_SHARE_FETCH_RESPONSE); + rko->rko_u.share_fetch_response.inflight_acks = rd_list_new(0, NULL); + rko->rko_u.share_fetch_response.message_rkos = rd_list_new(0, NULL); + return rko; +} + +/** + * @brief Set all entry types to a single uniform value. + */ +static void +ut_set_types_uniform(rd_kafka_share_ack_batch_entry_t *entry, + rd_kafka_share_internal_acknowledgement_type t) { + int i; + for (i = 0; i < entry->types_cnt; i++) + entry->types[i] = t; +} + +/** + * @brief Set entry types from an array. + */ +static void +ut_set_types(rd_kafka_share_ack_batch_entry_t *entry, + const rd_kafka_share_internal_acknowledgement_type *types, + int cnt) { + int i; + for (i = 0; i < cnt; i++) + entry->types[i] = types[i]; +} + +/**@}*/ + + +/** + * @name Tests for rd_kafka_share_filter_acquired_records_and_update_ack_type() + * @{ + */ + +/** @brief All messages in contiguous range are acquired with ACQUIRED type. */ +static int unittest_filter_all_acquired(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int64_t FirstOffsets[1] = {0}; + int64_t LastOffsets[1] = {4}; + int16_t DeliveryCounts[1] = {1}; + int i; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = ut_create_fetch_op_with_offset(i); + rd_kafka_q_enq(temp_fetchq, rko); + } + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, 1); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 5, + "Expected 5 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = rd_list_elem(&filtered_msgs, i); + rd_kafka_msg_t *rkm = &rko->rko_u.fetch.rkm; + RD_UT_ASSERT(rkm->rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + "Message %d: expected ACQUIRED ack_type", i); + RD_UT_ASSERT(rkm->rkm_u.consumer.delivery_count == 1, + "Message %d: expected delivery_count=1, got %d", i, + rkm->rkm_u.consumer.delivery_count); + } + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/** @brief Only messages within acquired range pass; others filtered out. */ +static int unittest_filter_partial_range(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int64_t FirstOffsets[1] = {2}; + int64_t LastOffsets[1] = {5}; + int16_t DeliveryCounts[1] = {1}; + int i; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = ut_create_fetch_op_with_offset(i); + rd_kafka_q_enq(temp_fetchq, rko); + } + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, 1); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 4, + "Expected 4 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + for (i = 0; i < 4; i++) { + rd_kafka_op_t *rko = rd_list_elem(&filtered_msgs, i); + int64_t offset = rd_kafka_op_get_offset(rko); + RD_UT_ASSERT(offset == 2 + i, + "Message %d: expected offset %d, got %" PRId64, i, + 2 + i, offset); + } + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/** @brief Multiple disjoint acquired ranges filter correct messages. */ +static int unittest_filter_disjoint_ranges(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int64_t FirstOffsets[3] = {1, 5, 9}; + int64_t LastOffsets[3] = {2, 6, 9}; + int16_t DeliveryCounts[3] = {1, 2, 3}; + int i; + int64_t expected_offsets[] = {1, 2, 5, 6, 9}; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = ut_create_fetch_op_with_offset(i); + rd_kafka_q_enq(temp_fetchq, rko); + } + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, 3); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 5, + "Expected 5 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = rd_list_elem(&filtered_msgs, i); + int64_t offset = rd_kafka_op_get_offset(rko); + RD_UT_ASSERT(offset == expected_offsets[i], + "Message %d: expected offset %" PRId64 + ", got %" PRId64, + i, expected_offsets[i], offset); + } + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/** @brief Empty queue returns no messages. */ +static int unittest_filter_empty_queue(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int64_t FirstOffsets[1] = {0}; + int64_t LastOffsets[1] = {10}; + int16_t DeliveryCounts[1] = {1}; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, 1); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 0, + "Expected 0 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/** @brief No acquired ranges means all messages filtered out. */ +static int unittest_filter_no_ranges(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int i; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = ut_create_fetch_op_with_offset(i); + rd_kafka_q_enq(temp_fetchq, rko); + } + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, NULL, NULL, NULL, 0); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 0, + "Expected 0 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/** @brief Non-sequential offsets in queue; only matching ones pass. */ +static int unittest_filter_sparse_offsets(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int64_t FirstOffsets[1] = {5}; + int64_t LastOffsets[1] = {25}; + int16_t DeliveryCounts[1] = {1}; + int64_t sparse_offsets[] = {0, 10, 20, 30}; + int i; + int64_t expected_offsets[] = {10, 20}; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + for (i = 0; i < 4; i++) { + rd_kafka_op_t *rko = + ut_create_fetch_op_with_offset(sparse_offsets[i]); + rd_kafka_q_enq(temp_fetchq, rko); + } + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, 1); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 2, + "Expected 2 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + for (i = 0; i < 2; i++) { + rd_kafka_op_t *rko = rd_list_elem(&filtered_msgs, i); + int64_t offset = rd_kafka_op_get_offset(rko); + RD_UT_ASSERT(offset == expected_offsets[i], + "Message %d: expected offset %" PRId64 + ", got %" PRId64, + i, expected_offsets[i], offset); + } + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/** @brief Acquired range beyond all queue messages returns empty. */ +static int unittest_filter_range_beyond_messages(void) { + rd_kafka_q_t *temp_fetchq = ut_create_mock_queue(); + rd_list_t filtered_msgs; + int64_t FirstOffsets[1] = {100}; + int64_t LastOffsets[1] = {200}; + int16_t DeliveryCounts[1] = {1}; + int i; + + rd_list_init(&filtered_msgs, 0, ut_op_destroy_free); + + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = ut_create_fetch_op_with_offset(i); + rd_kafka_q_enq(temp_fetchq, rko); + } + + rd_kafka_share_filter_acquired_records_and_update_ack_type( + temp_fetchq, &filtered_msgs, FirstOffsets, LastOffsets, + DeliveryCounts, 1); + + RD_UT_ASSERT(rd_list_cnt(&filtered_msgs) == 0, + "Expected 0 messages, got %d", + rd_list_cnt(&filtered_msgs)); + + rd_list_destroy(&filtered_msgs); + ut_destroy_mock_queue(temp_fetchq); + + RD_UT_PASS(); +} + +/**@}*/ + + +/** + * @name Tests for rd_kafka_share_build_ack_details() + * @{ + */ + +/** @brief All ACCEPT types extracted as single collated entry. */ +static int unittest_ack_details_all_accept(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + rd_kafka_share_ack_batch_entry_t *out_entry; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 5); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types_uniform(entry, RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + RD_UT_ASSERT(rd_list_cnt(ack_details) == 1, "Expected 1 batch, got %d", + rd_list_cnt(ack_details)); + + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(rd_list_cnt(&out_batch->entries) == 1, + "Expected 1 collated entry, got %d", + rd_list_cnt(&out_batch->entries)); + + out_entry = rd_list_elem(&out_batch->entries, 0); + RD_UT_ASSERT(out_entry->start_offset == 100, + "Expected start=100, got %" PRId64, + out_entry->start_offset); + RD_UT_ASSERT(out_entry->end_offset == 104, + "Expected end=104, got %" PRId64, out_entry->end_offset); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "Expected ACCEPT type"); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 0, + "Expected empty map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "Expected unacked_cnt=0, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief All ACQUIRED types returns NULL (nothing to extract). */ +static int unittest_ack_details_all_acquired(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 5); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types_uniform(entry, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + rkshare->rkshare_unacked_cnt = 5; + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details == NULL, "Expected NULL ack_details"); + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected map to still have 1 entry"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 5, + "Expected unacked_cnt=5, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Mixed types collated; ACQUIRED offsets remain in map. */ +static int unittest_ack_details_mixed_collation(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + rd_kafka_share_ack_batch_entry_t *out_entry; + const rd_kafka_share_internal_acknowledgement_type types[] = { + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT}; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 10); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 109, 10, 1); + ut_set_types(entry, types, 10); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + RD_UT_ASSERT(rd_list_cnt(ack_details) == 1, "Expected 1 batch"); + + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(rd_list_cnt(&out_batch->entries) == 4, + "Expected 4 collated entries, got %d", + rd_list_cnt(&out_batch->entries)); + + out_entry = rd_list_elem(&out_batch->entries, 0); + RD_UT_ASSERT(out_entry->start_offset == 100 && + out_entry->end_offset == 101, + "Entry 0: expected 100-101"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "Entry 0: expected ACCEPT"); + + out_entry = rd_list_elem(&out_batch->entries, 1); + RD_UT_ASSERT(out_entry->start_offset == 102 && + out_entry->end_offset == 104, + "Entry 1: expected 102-104"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Entry 1: expected REJECT"); + + out_entry = rd_list_elem(&out_batch->entries, 2); + RD_UT_ASSERT(out_entry->start_offset == 107 && + out_entry->end_offset == 108, + "Entry 2: expected 107-108"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "Entry 2: expected RELEASE"); + + out_entry = rd_list_elem(&out_batch->entries, 3); + RD_UT_ASSERT(out_entry->start_offset == 109 && + out_entry->end_offset == 109, + "Entry 3: expected 109-109"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "Entry 3: expected ACCEPT"); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected 1 entry remaining in map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 2, + "Expected unacked_cnt=2, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Alternating types produce separate entries (no collation). */ +static int unittest_ack_details_alternating(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + const rd_kafka_share_internal_acknowledgement_type types[] = { + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT}; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 5); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types(entry, types, 5); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(rd_list_cnt(&out_batch->entries) == 5, + "Expected 5 collated entries, got %d", + rd_list_cnt(&out_batch->entries)); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 0, + "Expected empty map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "Expected unacked_cnt=0"); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Multiple partitions with mixed states processed independently. */ +static int unittest_ack_details_multi_partition(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id0 = rd_kafka_Uuid_random(); + rd_kafka_Uuid_t topic_id1 = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + const rd_kafka_share_internal_acknowledgement_type types1[] = { + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT}; + + rd_kafka_topic_partition_t *rktpar0 = + ut_create_rktpar_with_id("test-topic", 0, topic_id0); + rd_kafka_share_ack_batches_t *batches0 = + rd_kafka_share_ack_batches_new(rktpar0, 1, 3); + rd_kafka_share_ack_batch_entry_t *entry0 = + rd_kafka_share_ack_batch_entry_new(100, 102, 3, 1); + ut_set_types_uniform(entry0, RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT); + rd_list_add(&batches0->entries, entry0); + ut_add_batches_to_map(rkshare, batches0); + + rd_kafka_topic_partition_t *rktpar1 = + ut_create_rktpar_with_id("test-topic", 1, topic_id1); + rd_kafka_share_ack_batches_t *batches1 = + rd_kafka_share_ack_batches_new(rktpar1, 2, 5); + rd_kafka_share_ack_batch_entry_t *entry1 = + rd_kafka_share_ack_batch_entry_new(200, 204, 5, 2); + ut_set_types(entry1, types1, 5); + rd_list_add(&batches1->entries, entry1); + ut_add_batches_to_map(rkshare, batches1); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + RD_UT_ASSERT(rd_list_cnt(ack_details) == 2, + "Expected 2 batches, got %d", rd_list_cnt(ack_details)); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected 1 entry in map (partition 1)"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 2, + "Expected unacked_cnt=2, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Empty inflight map returns NULL. */ +static int unittest_ack_details_empty_map(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_list_t *ack_details; + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details == NULL, "Expected NULL for empty map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "Expected unacked_cnt=0"); + + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Leader metadata and delivery_count preserved in output. */ +static int unittest_ack_details_metadata(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + rd_kafka_share_ack_batch_entry_t *out_entry; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 5, 2); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 101, 2, 3); + ut_set_types_uniform(entry, RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(out_batch->response_leader_id == 5, + "Expected leader_id=5, got %d", + out_batch->response_leader_id); + + out_entry = rd_list_elem(&out_batch->entries, 0); + RD_UT_ASSERT(out_entry->delivery_count == 3, + "Expected delivery_count=3, got %d", + out_entry->delivery_count); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief GAPs extracted while ACQUIRED offsets remain in map. */ +static int unittest_ack_details_gap_acquired(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + const rd_kafka_share_internal_acknowledgement_type types[] = { + RD_KAFKA_SHARE_INTERNAL_ACK_GAP, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_GAP, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_GAP}; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 5); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types(entry, types, 5); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + rkshare->rkshare_unacked_cnt = 2; + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + RD_UT_ASSERT(rd_list_cnt(ack_details) == 1, "Expected 1 batch"); + + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(rd_list_cnt(&out_batch->entries) == 3, + "Expected 3 GAP entries, got %d", + rd_list_cnt(&out_batch->entries)); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected 1 entry remaining in map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 2, + "Expected unacked_cnt=2, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Multiple batch entries in same partition processed independently. */ +static int unittest_ack_details_multi_entry(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + rd_kafka_share_ack_batch_entry_t *out_entry; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 6); + + rd_kafka_share_ack_batch_entry_t *entry1 = + rd_kafka_share_ack_batch_entry_new(100, 102, 3, 1); + ut_set_types_uniform(entry1, RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT); + rd_list_add(&batches->entries, entry1); + + rd_kafka_share_ack_batch_entry_t *entry2 = + rd_kafka_share_ack_batch_entry_new(200, 202, 3, 2); + ut_set_types_uniform(entry2, RD_KAFKA_SHARE_INTERNAL_ACK_REJECT); + rd_list_add(&batches->entries, entry2); + + ut_add_batches_to_map(rkshare, batches); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + RD_UT_ASSERT(rd_list_cnt(ack_details) == 1, "Expected 1 batch"); + + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(rd_list_cnt(&out_batch->entries) == 2, + "Expected 2 collated entries, got %d", + rd_list_cnt(&out_batch->entries)); + + out_entry = rd_list_elem(&out_batch->entries, 0); + RD_UT_ASSERT(out_entry->start_offset == 100 && + out_entry->end_offset == 102, + "Entry 0: expected 100-102"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "Entry 0: expected ACCEPT"); + + out_entry = rd_list_elem(&out_batch->entries, 1); + RD_UT_ASSERT(out_entry->start_offset == 200 && + out_entry->end_offset == 202, + "Entry 1: expected 200-202"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Entry 1: expected REJECT"); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 0, + "Expected empty map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "Expected unacked_cnt=0"); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Interleaved ACQUIRED breaks extractable ranges into segments. */ +static int unittest_ack_details_interleaved(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_list_t *ack_details; + rd_kafka_share_ack_batches_t *out_batch; + rd_kafka_share_ack_batch_entry_t *out_entry; + const rd_kafka_share_internal_acknowledgement_type types[] = { + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT}; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 10); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 109, 10, 1); + ut_set_types(entry, types, 10); + rd_list_add(&batches->entries, entry); + + ut_add_batches_to_map(rkshare, batches); + + ack_details = rd_kafka_share_build_ack_details(rkshare); + + RD_UT_ASSERT(ack_details != NULL, "Expected non-NULL ack_details"); + out_batch = rd_list_elem(ack_details, 0); + RD_UT_ASSERT(rd_list_cnt(&out_batch->entries) == 4, + "Expected 4 collated entries, got %d", + rd_list_cnt(&out_batch->entries)); + + out_entry = rd_list_elem(&out_batch->entries, 0); + RD_UT_ASSERT(out_entry->start_offset == 100 && + out_entry->end_offset == 100, + "Entry 0: expected 100-100, got %" PRId64 "-%" PRId64, + out_entry->start_offset, out_entry->end_offset); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "Entry 0: expected ACCEPT"); + + out_entry = rd_list_elem(&out_batch->entries, 1); + RD_UT_ASSERT(out_entry->start_offset == 102 && + out_entry->end_offset == 103, + "Entry 1: expected 102-103"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Entry 1: expected REJECT"); + + out_entry = rd_list_elem(&out_batch->entries, 2); + RD_UT_ASSERT(out_entry->start_offset == 105 && + out_entry->end_offset == 106, + "Entry 2: expected 105-106"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "Entry 2: expected RELEASE"); + + out_entry = rd_list_elem(&out_batch->entries, 3); + RD_UT_ASSERT(out_entry->start_offset == 108 && + out_entry->end_offset == 109, + "Entry 3: expected 108-109"); + RD_UT_ASSERT(out_entry->types[0] == RD_KAFKA_SHARE_INTERNAL_ACK_ACCEPT, + "Entry 3: expected ACCEPT"); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected 1 entry remaining in map"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 3, + "Expected unacked_cnt=3, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_list_destroy(ack_details); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/**@}*/ + + +/** + * @name Tests for rd_kafka_share_build_inflight_acks_map() + * @{ + */ + +/** @brief Single partition with all ACQUIRED mapped correctly. */ +static int unittest_ack_mapping_single_partition(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_kafka_op_t *response_rko = ut_create_share_fetch_response_rko(); + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 5); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types_uniform(entry, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches->entries, entry); + + rd_list_add(response_rko->rko_u.share_fetch_response.inflight_acks, + batches); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected 1 map entry, got %d", + (int)RD_MAP_CNT(&rkshare->rkshare_inflight_acks)); + + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 5, + "Expected unacked_cnt=5, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_kafka_op_destroy(response_rko); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Multiple partitions create separate map entries. */ +static int unittest_ack_mapping_multi_partition(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id0 = rd_kafka_Uuid_random(); + rd_kafka_Uuid_t topic_id1 = rd_kafka_Uuid_random(); + rd_kafka_op_t *response_rko = ut_create_share_fetch_response_rko(); + + rd_kafka_topic_partition_t *rktpar0 = + ut_create_rktpar_with_id("test-topic", 0, topic_id0); + rd_kafka_share_ack_batches_t *batches0 = + rd_kafka_share_ack_batches_new(rktpar0, 1, 3); + rd_kafka_share_ack_batch_entry_t *entry0 = + rd_kafka_share_ack_batch_entry_new(100, 102, 3, 1); + ut_set_types_uniform(entry0, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches0->entries, entry0); + rd_list_add(response_rko->rko_u.share_fetch_response.inflight_acks, + batches0); + + rd_kafka_topic_partition_t *rktpar1 = + ut_create_rktpar_with_id("test-topic", 1, topic_id1); + rd_kafka_share_ack_batches_t *batches1 = + rd_kafka_share_ack_batches_new(rktpar1, 2, 2); + rd_kafka_share_ack_batch_entry_t *entry1 = + rd_kafka_share_ack_batch_entry_new(200, 201, 2, 1); + ut_set_types_uniform(entry1, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches1->entries, entry1); + rd_list_add(response_rko->rko_u.share_fetch_response.inflight_acks, + batches1); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 2, + "Expected 2 map entries, got %d", + (int)RD_MAP_CNT(&rkshare->rkshare_inflight_acks)); + + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 5, + "Expected unacked_cnt=5, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_kafka_op_destroy(response_rko); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief GAP types not counted in unacked_cnt; only ACQUIRED counted. */ +static int unittest_ack_mapping_gaps(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_kafka_op_t *response_rko = ut_create_share_fetch_response_rko(); + const rd_kafka_share_internal_acknowledgement_type types[] = { + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_GAP, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED, + RD_KAFKA_SHARE_INTERNAL_ACK_GAP, + RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED}; + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 5); + + rd_kafka_share_ack_batch_entry_t *entry = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types(entry, types, 5); + rd_list_add(&batches->entries, entry); + + rd_list_add(response_rko->rko_u.share_fetch_response.inflight_acks, + batches); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko); + + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 3, + "Expected unacked_cnt=3, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_kafka_op_destroy(response_rko); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Empty inflight_acks list leaves map empty. */ +static int unittest_ack_mapping_empty(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_op_t *response_rko = ut_create_share_fetch_response_rko(); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 0, + "Expected empty map"); + + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 0, + "Expected unacked_cnt=0"); + + rd_kafka_op_destroy(response_rko); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Multiple entries per partition stored in single map entry. */ +static int unittest_ack_mapping_multi_entry(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id = rd_kafka_Uuid_random(); + rd_kafka_op_t *response_rko = ut_create_share_fetch_response_rko(); + + rd_kafka_topic_partition_t *rktpar = + ut_create_rktpar_with_id("test-topic", 0, topic_id); + rd_kafka_share_ack_batches_t *batches = + rd_kafka_share_ack_batches_new(rktpar, 1, 7); + + rd_kafka_share_ack_batch_entry_t *entry1 = + rd_kafka_share_ack_batch_entry_new(100, 104, 5, 1); + ut_set_types_uniform(entry1, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches->entries, entry1); + + rd_kafka_share_ack_batch_entry_t *entry2 = + rd_kafka_share_ack_batch_entry_new(200, 201, 2, 2); + ut_set_types_uniform(entry2, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches->entries, entry2); + + rd_list_add(response_rko->rko_u.share_fetch_response.inflight_acks, + batches); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "Expected 1 map entry"); + + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 7, + "Expected unacked_cnt=7, got %" PRId64, + rkshare->rkshare_unacked_cnt); + + rd_kafka_op_destroy(response_rko); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/** @brief Multiple calls accumulate partitions in map. */ +static int unittest_ack_mapping_cumulative(void) { + rd_kafka_share_t *rkshare = ut_create_mock_rkshare(); + rd_kafka_Uuid_t topic_id0 = rd_kafka_Uuid_random(); + rd_kafka_Uuid_t topic_id1 = rd_kafka_Uuid_random(); + rd_kafka_op_t *response_rko1, *response_rko2; + + response_rko1 = ut_create_share_fetch_response_rko(); + rd_kafka_topic_partition_t *rktpar0 = + ut_create_rktpar_with_id("test-topic", 0, topic_id0); + rd_kafka_share_ack_batches_t *batches0 = + rd_kafka_share_ack_batches_new(rktpar0, 1, 3); + rd_kafka_share_ack_batch_entry_t *entry0 = + rd_kafka_share_ack_batch_entry_new(100, 102, 3, 1); + ut_set_types_uniform(entry0, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches0->entries, entry0); + rd_list_add(response_rko1->rko_u.share_fetch_response.inflight_acks, + batches0); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko1); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 1, + "After 1st call: expected 1 map entry"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 3, + "After 1st call: expected unacked_cnt=3"); + + response_rko2 = ut_create_share_fetch_response_rko(); + rd_kafka_topic_partition_t *rktpar1 = + ut_create_rktpar_with_id("test-topic", 1, topic_id1); + rd_kafka_share_ack_batches_t *batches1 = + rd_kafka_share_ack_batches_new(rktpar1, 2, 2); + rd_kafka_share_ack_batch_entry_t *entry1 = + rd_kafka_share_ack_batch_entry_new(200, 201, 2, 1); + ut_set_types_uniform(entry1, RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED); + rd_list_add(&batches1->entries, entry1); + rd_list_add(response_rko2->rko_u.share_fetch_response.inflight_acks, + batches1); + + rd_kafka_share_build_inflight_acks_map(rkshare, response_rko2); + + RD_UT_ASSERT(RD_MAP_CNT(&rkshare->rkshare_inflight_acks) == 2, + "After 2nd call: expected 2 map entries"); + RD_UT_ASSERT(rkshare->rkshare_unacked_cnt == 5, + "After 2nd call: expected unacked_cnt=5"); + + rd_kafka_op_destroy(response_rko1); + rd_kafka_op_destroy(response_rko2); + ut_destroy_mock_rkshare(rkshare); + + RD_UT_PASS(); +} + +/**@}*/ + + +/** + * @name Main test entry point + * @{ + */ + +int unittest_fetcher_share_filter_forward(void) { + int fails = 0; + const struct { + const char *name; + int (*call)(void); + } tests[] = { + /* filter tests */ + {"filter_all_acquired", unittest_filter_all_acquired}, + {"filter_partial_range", unittest_filter_partial_range}, + {"filter_disjoint_ranges", unittest_filter_disjoint_ranges}, + {"filter_empty_queue", unittest_filter_empty_queue}, + {"filter_no_ranges", unittest_filter_no_ranges}, + {"filter_sparse_offsets", unittest_filter_sparse_offsets}, + {"filter_range_beyond", unittest_filter_range_beyond_messages}, + /* ack_details tests */ + {"ack_details_all_accept", unittest_ack_details_all_accept}, + {"ack_details_all_acquired", unittest_ack_details_all_acquired}, + {"ack_details_mixed_collation", + unittest_ack_details_mixed_collation}, + {"ack_details_alternating", unittest_ack_details_alternating}, + {"ack_details_multi_partition", + unittest_ack_details_multi_partition}, + {"ack_details_empty_map", unittest_ack_details_empty_map}, + {"ack_details_metadata", unittest_ack_details_metadata}, + {"ack_details_gap_acquired", unittest_ack_details_gap_acquired}, + {"ack_details_multi_entry", unittest_ack_details_multi_entry}, + {"ack_details_interleaved", unittest_ack_details_interleaved}, + /* ack_mapping tests */ + {"ack_mapping_single_partition", + unittest_ack_mapping_single_partition}, + {"ack_mapping_multi_partition", + unittest_ack_mapping_multi_partition}, + {"ack_mapping_gaps", unittest_ack_mapping_gaps}, + {"ack_mapping_empty", unittest_ack_mapping_empty}, + {"ack_mapping_multi_entry", unittest_ack_mapping_multi_entry}, + {"ack_mapping_cumulative", unittest_ack_mapping_cumulative}, + {NULL}}; + int i; + + ut_mock_rk = rd_calloc(1, sizeof(*ut_mock_rk)); + + for (i = 0; tests[i].name; i++) { + int f = tests[i].call(); + RD_UT_SAY(" %s: %s", tests[i].name, + f ? "\033[31mFAIL\033[0m" : "\033[32mPASS\033[0m"); + fails += f; + } + + rd_free(ut_mock_rk); + ut_mock_rk = NULL; + + if (fails > 0) { + RD_UT_SAY("%d test(s) failed", fails); + return 1; + } + + RD_UT_PASS(); +} + +/**@}*/ diff --git a/lib/librdkafka-2.15.0/src/rdunittest_msgset_errors.c b/lib/librdkafka-2.15.0/src/rdunittest_msgset_errors.c new file mode 100644 index 00000000000..8d154fe7815 --- /dev/null +++ b/lib/librdkafka-2.15.0/src/rdunittest_msgset_errors.c @@ -0,0 +1,1521 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @name Unit tests for Share Consumer MessageSet error handling + * @{ + * + * Tests cover per-offset error handling for: + * - CRC errors → REJECT ack_type + * - Decompression errors → RELEASE ack_type + * - Unsupported MagicByte → REJECT ack_type + * + * These tests craft binary MessageSet v2 buffers with intentional corruption + * and verify that the msgset reader creates one error op per offset with + * correct ack_types for share consumers. + */ + +#include "rd.h" +#include "rdunittest.h" +#include "rdkafka_int.h" +#include "rdkafka_queue.h" +#include "rdkafka_partition.h" +#include "rdkafka_share_acknowledgement.h" +#include "rdkafka_buf.h" +#include "rdkafka_msgset.h" +#include "crc32c.h" + +#if WITH_ZSTD +#include "rdkafka_zstd.h" +#endif + + +/** + * @brief Alignment-safe big-endian writers. + * + * The msgset crafting helpers below write into a char buffer at arbitrary + * offsets. Casting `char *` to `int{16,32,64}_t *` triggers -Wcast-align + * on stricter targets (and is undefined behaviour even on x86 strictly + * speaking). Use memcpy() — the compiler lowers it to a single move on + * supported architectures. + */ +static RD_INLINE void ut_put_be16(void *p, int16_t v) { + int16_t be = htobe16(v); + memcpy(p, &be, sizeof(be)); +} +static RD_INLINE void ut_put_be32(void *p, int32_t v) { + int32_t be = htobe32(v); + memcpy(p, &be, sizeof(be)); +} +static RD_INLINE void ut_put_be64(void *p, int64_t v) { + int64_t be = htobe64(v); + memcpy(p, &be, sizeof(be)); +} + + +/** + * @name Test helpers + * @{ + */ + +/** + * @brief Create a minimal share consumer for testing + */ +static rd_kafka_share_t *ut_create_test_share_consumer(void) { + rd_kafka_conf_t *conf = rd_kafka_conf_new(); + rd_kafka_share_t *rkshare; + char errstr[512]; + + if (rd_kafka_conf_set(conf, "group.id", "ut-msgset-errors", errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return NULL; + } + + /* Enable CRC checking for tests */ + if (rd_kafka_conf_set(conf, "check.crcs", "true", errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return NULL; + } + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + return rkshare; +} + +static void ut_destroy_share_consumer(rd_kafka_share_t *rkshare) { + if (!rkshare) + return; + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); +} + +/** + * @brief Create a mock topic partition + */ +static rd_kafka_toppar_t * +ut_create_mock_toppar(rd_kafka_t *rk, const char *topic, int32_t partition) { + rd_kafka_topic_t *rkt = rd_kafka_topic_new(rk, topic, NULL); + if (!rkt) + return NULL; + return rd_kafka_toppar_new(rkt, partition); +} + +/** + * @brief Helper to write MessageSet v2 header fields in big-endian + */ +static void ut_write_msgset_v2_header(char *buf, + int64_t BaseOffset, + int32_t Length, + int32_t PartitionLeaderEpoch, + int8_t MagicByte, + int32_t Crc, + int16_t Attributes, + int32_t LastOffsetDelta, + int64_t BaseTimestamp, + int64_t MaxTimestamp, + int64_t ProducerId, + int16_t ProducerEpoch, + int32_t BaseSequence, + int32_t RecordCount) { + size_t offset = 0; + + /* BaseOffset (int64) */ + ut_put_be64(buf + offset, BaseOffset); + offset += 8; + + /* Length (int32) */ + ut_put_be32(buf + offset, Length); + offset += 4; + + /* PartitionLeaderEpoch (int32) */ + ut_put_be32(buf + offset, PartitionLeaderEpoch); + offset += 4; + + /* MagicByte (int8) */ + *(int8_t *)(buf + offset) = MagicByte; + offset += 1; + + /* Crc (int32) */ + ut_put_be32(buf + offset, Crc); + offset += 4; + + /* Attributes (int16) */ + ut_put_be16(buf + offset, Attributes); + offset += 2; + + /* LastOffsetDelta (int32) */ + ut_put_be32(buf + offset, LastOffsetDelta); + offset += 4; + + /* BaseTimestamp (int64) */ + ut_put_be64(buf + offset, BaseTimestamp); + offset += 8; + + /* MaxTimestamp (int64) */ + ut_put_be64(buf + offset, MaxTimestamp); + offset += 8; + + /* ProducerId (int64) */ + ut_put_be64(buf + offset, ProducerId); + offset += 8; + + /* ProducerEpoch (int16) */ + ut_put_be16(buf + offset, ProducerEpoch); + offset += 2; + + /* BaseSequence (int32) */ + ut_put_be32(buf + offset, BaseSequence); + offset += 4; + + /* RecordCount (int32) */ + ut_put_be32(buf + offset, RecordCount); + offset += 4; +} + +#if WITH_ZSTD +/** + * @brief Calculate correct CRC32C for MessageSet v2 + * CRC covers from Attributes to end of records + */ +static uint32_t ut_calc_msgset_crc(const char *buf, + size_t offset_to_attributes, + size_t total_len) { + /* CRC starts after MagicByte and Crc field itself */ + const void *data = buf + offset_to_attributes; + size_t data_len = total_len - offset_to_attributes; + return rd_crc32c(0, data, data_len); +} +#endif /* WITH_ZSTD */ + +/** + * @brief Write a varint (used in Record format) + * @returns number of bytes written + */ +static size_t ut_write_varint(char *buf, int64_t value) { + uint64_t uval = (value << 1) ^ (value >> 63); /* Zigzag encoding */ + size_t i = 0; + + while (uval >= 0x80) { + buf[i++] = (char)((uval & 0x7F) | 0x80); + uval >>= 7; + } + buf[i++] = (char)(uval & 0x7F); + return i; +} + +/** + * @brief Build a minimal valid Record v2 + * @returns size of record written + */ +static size_t ut_build_record_v2(char *buf, + int32_t offset_delta, + int64_t timestamp_delta, + const char *key, + size_t key_len, + const char *value, + size_t value_len) { + size_t offset = 0; + size_t len_offset; + int32_t key_len_varint = key ? (int32_t)key_len : -1; + int32_t val_len_varint = value ? (int32_t)value_len : -1; + + /* Reserve space for Length (varint) - we'll fill this in later */ + len_offset = offset; + offset += 5; /* Max varint size for record length */ + + /* Attributes (int8) */ + buf[offset++] = 0; + + /* TimestampDelta (varint) */ + offset += ut_write_varint(buf + offset, timestamp_delta); + + /* OffsetDelta (varint) */ + offset += ut_write_varint(buf + offset, offset_delta); + + /* KeyLen (varint) */ + offset += ut_write_varint(buf + offset, key_len_varint); + + /* Key bytes */ + if (key) { + memcpy(buf + offset, key, key_len); + offset += key_len; + } + + /* ValueLen (varint) */ + offset += ut_write_varint(buf + offset, val_len_varint); + + /* Value bytes */ + if (value) { + memcpy(buf + offset, value, value_len); + offset += value_len; + } + + /* Headers count (varint) - 0 headers */ + offset += ut_write_varint(buf + offset, 0); + + /* Calculate record length (everything after Length field) */ + size_t record_body_len = offset - len_offset - 5; + + /* Write actual Length at the beginning */ + size_t len_size = ut_write_varint(buf + len_offset, record_body_len); + + /* Compact the record if varint was smaller than 5 bytes */ + if (len_size < 5) { + memmove(buf + len_offset + len_size, buf + len_offset + 5, + record_body_len); + offset = len_offset + len_size + record_body_len; + } + + return offset; +} + +/** + * @brief Build a complete valid MessageSet v2 with records + * @returns total size of MessageSet + */ +static size_t ut_build_valid_msgset_v2(char *buf, + int64_t base_offset, + int32_t record_count, + const char **values, + size_t *value_lens) { + size_t offset = 0; + size_t len_offset = 8; + size_t records_size = 0; + int32_t i; + char *records_buf; + uint32_t crc; + int32_t last_offset_delta = record_count > 0 ? record_count - 1 : 0; + + /* Build all records first to know total size */ + records_buf = rd_calloc(1, 8192); + for (i = 0; i < record_count; i++) { + records_size += ut_build_record_v2( + records_buf + records_size, i, /* offsetDelta */ + 0, /* timestampDelta */ + NULL, 0, /* no key */ + values ? values[i] : NULL, + values && value_lens ? value_lens[i] : 0); + } + + /* BaseOffset */ + ut_put_be64(buf + offset, base_offset); + offset += 8; + + /* Length - will update later */ + len_offset = offset; + offset += 4; + + /* PartitionLeaderEpoch */ + ut_put_be32(buf + offset, -1); + offset += 4; + + /* MagicByte */ + buf[offset++] = 2; + + /* CRC - placeholder, will calculate later */ + size_t crc_offset = offset; + offset += 4; + + /* Attributes (no compression) */ + ut_put_be16(buf + offset, 0); + offset += 2; + + /* LastOffsetDelta */ + ut_put_be32(buf + offset, last_offset_delta); + offset += 4; + + /* BaseTimestamp */ + ut_put_be64(buf + offset, 0); + offset += 8; + + /* MaxTimestamp */ + ut_put_be64(buf + offset, 0); + offset += 8; + + /* ProducerId */ + ut_put_be64(buf + offset, -1); + offset += 8; + + /* ProducerEpoch */ + ut_put_be16(buf + offset, -1); + offset += 2; + + /* BaseSequence */ + ut_put_be32(buf + offset, -1); + offset += 4; + + /* RecordCount */ + ut_put_be32(buf + offset, record_count); + offset += 4; + + /* Copy records */ + memcpy(buf + offset, records_buf, records_size); + offset += records_size; + rd_free(records_buf); + + /* Calculate and write Length */ + int32_t length = (int32_t)(offset - len_offset - 4); + ut_put_be32(buf + len_offset, length); + + /* Calculate and write CRC (covers Attributes to end) */ + crc = rd_crc32c(0, buf + crc_offset + 4, offset - crc_offset - 4); + ut_put_be32(buf + crc_offset, crc); + + return offset; +} + +/**@}*/ + + +/** + * @brief Test CRC error generates per-offset error ops with REJECT ack_type + * + * Creates a MessageSet v2 with intentionally wrong CRC and verifies: + * - One error op per offset in range [BaseOffset, LastOffset] + * - Each error op has REJECT ack_type + * - Error code is RD_KAFKA_RESP_ERR__BAD_MSG + */ +static int unittest_msgset_crc_error_share_consumer(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *msgset_data; + size_t msgset_size; + int64_t BaseOffset = 100; + int32_t LastOffsetDelta = + 4; /* Offsets 100-104 inclusive = 5 messages */ + int expected_err_ops = 5; + int i; + struct rd_kafka_toppar_ver tver = {.version = 11}; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-crc", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + + /* Craft MessageSet v2 with wrong CRC */ + /* MessageSet v2 structure: + * - BaseOffset (8) + Length (4) = 12 bytes before "Length" data starts + * - Length field contains size of: PartitionLeaderEpoch (4) + MagicByte + * (1) + * + Crc (4) + Attributes (2) + LastOffsetDelta (4) + BaseTimestamp + * (8) + * + MaxTimestamp (8) + ProducerId (8) + ProducerEpoch (2) + * + BaseSequence (4) + RecordCount (4) + Records + * - Minimum Length = 49 (just the header fields after Length) + * - For this test, we need enough space for CRC validation + */ + + /* CRC validation needs: crc_len = Length - 4 - 1 - 4 bytes available + * after reading up to Crc field (21 bytes read so far) */ + int32_t Length = 61 - 12; /* Minimum: just the header, no records */ + msgset_size = 12 + Length; /* BaseOffset + Length + data */ + msgset_data = rd_calloc(1, msgset_size); + + ut_write_msgset_v2_header( + msgset_data, BaseOffset, + Length, /* Length of data after BaseOffset+Length fields */ + -1, /* PartitionLeaderEpoch */ + 2, /* MagicByte v2 */ + 0xDEADBEEF, /* WRONG CRC - will trigger error */ + 0, /* Attributes: no compression */ + LastOffsetDelta, 0, /* BaseTimestamp */ + 0, /* MaxTimestamp */ + -1, /* ProducerId */ + -1, /* ProducerEpoch */ + -1, /* BaseSequence */ + 5 /* RecordCount */); + + /* Create shadow buffer from raw data */ + rkbuf = rd_kafka_buf_new_shadow(msgset_data, msgset_size, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + /* Parse the MessageSet - should detect CRC error */ + rd_kafka_resp_err_t parse_err = + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + RD_UT_ASSERT(parse_err == RD_KAFKA_RESP_ERR_NO_ERROR || + parse_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Unexpected parse error: %s", rd_kafka_err2str(parse_err)); + + /* Verify we got expected number of error ops */ + for (i = 0; i < expected_err_ops; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, + "Expected error op %d of %d, got timeout", i + 1, + expected_err_ops); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG error, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == BaseOffset + i, + "Expected offset %" PRId64 ", got %" PRId64, + BaseOffset + i, rko->rko_u.err.offset); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT ack_type for CRC error, got %d", + rko->rko_u.err.rkm.rkm_u.consumer.ack_type); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko; + rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Got unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} + + +#if WITH_ZSTD +/** + * @brief Test decompression error generates per-offset error ops with RELEASE + * + * Creates a MessageSet v2 with ZSTD compression but corrupted compressed data. + * Verifies: + * - One error op per offset in range [BaseOffset, LastOffset] + * - Each error op has RELEASE ack_type (retryable error) + * - Error code is RD_KAFKA_RESP_ERR__BAD_COMPRESSION + */ +static int unittest_msgset_decompression_error_share_consumer(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *msgset_data; + size_t msgset_size; + int64_t BaseOffset = 200; + int32_t LastOffsetDelta = + 2; /* Offsets 200-202 inclusive = 3 messages */ + int expected_err_ops = 3; + int i; + uint32_t correct_crc; + size_t crc_offset = 17; /* Offset to CRC field in header */ + size_t attr_offset = + 21; /* Offset to Attributes field (where CRC starts) */ + struct rd_kafka_toppar_ver tver = {.version = 11}; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = ut_create_mock_toppar(rkshare->rkshare_rk, + "test-topic-decompress", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + + /* Craft MessageSet v2 with ZSTD compression but corrupted data */ + msgset_size = 200; + msgset_data = rd_calloc(1, msgset_size); + + int32_t Length = msgset_size - 12; + + ut_write_msgset_v2_header( + msgset_data, BaseOffset, + Length, /* Length of data after BaseOffset+Length fields */ + -1, /* PartitionLeaderEpoch */ + 2, /* MagicByte v2 */ + 0, /* CRC - will calculate correct one below */ + 4, /* Attributes: ZSTD compression (codec=4) */ + LastOffsetDelta, 0, /* BaseTimestamp */ + 0, /* MaxTimestamp */ + -1, /* ProducerId */ + -1, /* ProducerEpoch */ + -1, /* BaseSequence */ + 3 /* RecordCount */); + + /* Fill records section with garbage (corrupted ZSTD data) */ + memset(msgset_data + 61, 0xAB, msgset_size - 61); + + /* Calculate CORRECT CRC so we pass CRC check but fail decompression */ + correct_crc = ut_calc_msgset_crc(msgset_data, attr_offset, msgset_size); + ut_put_be32(msgset_data + crc_offset, correct_crc); + + /* Create shadow buffer from raw data */ + rkbuf = rd_kafka_buf_new_shadow(msgset_data, msgset_size, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + /* Parse the MessageSet - should detect decompression error */ + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify we got expected number of error ops */ + for (i = 0; i < expected_err_ops; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, + "Expected error op %d of %d, got timeout", i + 1, + expected_err_ops); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_COMPRESSION, + "Expected BAD_COMPRESSION error, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == BaseOffset + i, + "Expected offset %" PRId64 ", got %" PRId64, + BaseOffset + i, rko->rko_u.err.offset); + RD_UT_ASSERT( + rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "Expected RELEASE ack_type for decompression error, got %d", + rko->rko_u.err.rkm.rkm_u.consumer.ack_type); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} +#endif /* WITH_ZSTD */ + + +/** + * @brief Test unsupported MagicByte generates per-offset error ops with REJECT + * + * Creates a MessageSet with MagicByte=99 (unsupported future version). + * Verifies: + * - One error op per offset in range [BaseOffset, LastOffset] + * - Each error op has REJECT ack_type (permanent error) + * - Error code is RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED + */ +static int unittest_msgset_unsupported_magic_share_consumer(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *msgset_data; + size_t msgset_size; + int64_t BaseOffset = 300; + int32_t LastOffsetDelta = + 9; /* Offsets 300-309 inclusive = 10 messages */ + int expected_err_ops = 10; + int i; + struct rd_kafka_toppar_ver tver = {.version = 11}; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = + ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-magic", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + + /* Craft MessageSet with unsupported MagicByte */ + msgset_size = 200; + msgset_data = rd_calloc(1, msgset_size); + + int32_t Length = msgset_size - 12; + + ut_write_msgset_v2_header( + msgset_data, BaseOffset, + Length, /* Length of data after BaseOffset+Length fields */ + -1, /* PartitionLeaderEpoch */ + 99, /* UNSUPPORTED MagicByte - will trigger error */ + 0, /* CRC - doesn't matter, won't get that far */ + 0, /* Attributes */ + LastOffsetDelta, 0, /* BaseTimestamp */ + 0, /* MaxTimestamp */ + -1, /* ProducerId */ + -1, /* ProducerEpoch */ + -1, /* BaseSequence */ + 10 /* RecordCount */); + + /* Create shadow buffer from raw data */ + rkbuf = rd_kafka_buf_new_shadow(msgset_data, msgset_size, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + /* Parse the MessageSet - should detect unsupported MagicByte */ + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify we got expected number of error ops */ + for (i = 0; i < expected_err_ops; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, + "Expected error op %d of %d, got timeout", i + 1, + expected_err_ops); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, + "Expected NOT_IMPLEMENTED error, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == BaseOffset + i, + "Expected offset %" PRId64 ", got %" PRId64, + BaseOffset + i, rko->rko_u.err.offset); + RD_UT_ASSERT( + rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT ack_type for unsupported MagicByte, got " + "%d", + rko->rko_u.err.rkm.rkm_u.consumer.ack_type); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} + + +/** + * @brief Test mixed scenario: Success → CRC Error → Success + * + * Verifies parser correctly handles: + * - MessageSet 1: 3 successful messages (offsets 100-102) + * - MessageSet 2: CRC error covering 5 messages (offsets 103-107) + * - MessageSet 3: 2 successful messages (offsets 108-109) + */ +static int unittest_msgset_mixed_success_crc_success(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *buffer; + size_t buffer_size = 4096; + size_t offset = 0; + struct rd_kafka_toppar_ver tver = {.version = 11}; + const char *values[] = {"msg1", "msg2", "msg3", "msg4", "msg5"}; + size_t value_lens[] = {4, 4, 4, 4, 4}; + int i; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = + ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-mixed", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + buffer = rd_calloc(1, buffer_size); + + /* MessageSet 1: Valid with 3 messages (offsets 100-102) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 100, 3, values, + value_lens); + + /* MessageSet 2: CRC error with 5 messages (offsets 103-107) */ + int64_t BaseOffset2 = 103; + int32_t LastOffsetDelta2 = 4; + int32_t Length2 = 61 - 12; /* Just header */ + ut_write_msgset_v2_header(buffer + offset, BaseOffset2, Length2, -1, 2, + 0xBADC0C, /* Wrong CRC */ + 0, LastOffsetDelta2, 0, 0, -1, -1, -1, 5); + offset += 12 + Length2; + + /* MessageSet 3: Valid with 2 messages (offsets 108-109) */ + const char *values3[] = {"msg6", "msg7"}; + size_t value_lens3[] = {4, 4}; + offset += ut_build_valid_msgset_v2(buffer + offset, 108, 2, values3, + value_lens3); + + /* Parse all MessageSets */ + rkbuf = rd_kafka_buf_new_shadow(buffer, offset, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify MessageSet 1: 3 FETCH ops with offsets 100-102 */ + for (i = 0; i < 3; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op %d, got timeout", + i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 100 + i, + "Expected offset %d, got %" PRId64, 100 + i, + rko->rko_u.fetch.rkm.rkm_offset); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 2: 5 CONSUMER_ERR ops with offsets 103-107 */ + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, + "Expected CRC error op %d, got timeout", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG error, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == 103 + i, + "Expected offset %d, got %" PRId64, 103 + i, + rko->rko_u.err.offset); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT ack_type"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 3: 2 FETCH ops with offsets 108-109 */ + for (i = 0; i < 2; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op %d, got timeout", + i + 3); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 108 + i, + "Expected offset %d, got %" PRId64, 108 + i, + rko->rko_u.fetch.rkm.rkm_offset); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} + + +#if WITH_ZSTD +/** + * @brief Test mixed scenario: CRC Error → Success → Decompression Error + * + * Verifies parser correctly handles: + * - MessageSet 1: CRC error covering 2 messages (offsets 200-201) + * - MessageSet 2: 4 successful messages (offsets 202-205) + * - MessageSet 3: Decompression error covering 3 messages (offsets 206-208) + */ +static int unittest_msgset_mixed_crc_success_decomp(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *buffer; + size_t buffer_size = 4096; + size_t offset = 0; + struct rd_kafka_toppar_ver tver = {.version = 11}; + const char *values[] = {"a", "b", "c", "d"}; + size_t value_lens[] = {1, 1, 1, 1}; + int i; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = + ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-mixed2", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + buffer = rd_calloc(1, buffer_size); + + /* MessageSet 1: CRC error with 2 messages (offsets 200-201) */ + ut_write_msgset_v2_header(buffer + offset, 200, 61 - 12, -1, 2, + 0xDEADC0C, 0, 1, 0, 0, -1, -1, -1, 2); + offset += 12 + (61 - 12); + + /* MessageSet 2: Valid with 4 messages (offsets 202-205) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 202, 4, values, + value_lens); + + /* MessageSet 3: Decompression error with 3 messages (offsets 206-208) + */ + char corrupted_data[16] = {0xDE, 0xAD, 0xBE, 0xEF}; /* Garbage */ + size_t decomp_start = offset; + ut_write_msgset_v2_header(buffer + offset, 206, + 49 + sizeof(corrupted_data), -1, 2, 0, + 4, /* ZSTD compression */ + 2, 0, 0, -1, -1, -1, 3); + offset += 61; + memcpy(buffer + offset, corrupted_data, sizeof(corrupted_data)); + offset += sizeof(corrupted_data); + /* Fix CRC for decompression error (CRC is valid, data is corrupt) */ + uint32_t crc = rd_crc32c(0, buffer + decomp_start + 21, + 49 + sizeof(corrupted_data) - 4 - 1 - 4); + ut_put_be32(buffer + decomp_start + 17, crc); + + /* Parse all MessageSets */ + rkbuf = rd_kafka_buf_new_shadow(buffer, offset, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify MessageSet 1: 2 CRC error ops (offsets 200-201) */ + for (i = 0; i < 2; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected CRC error op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG"); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 2: 4 successful FETCH ops (offsets 202-205) */ + for (i = 0; i < 4; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op"); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 202 + i, + "Expected offset %d", 202 + i); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 3: 3 decompression error ops (offsets 206-208) */ + for (i = 0; i < 3; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected decomp error op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_COMPRESSION, + "Expected BAD_COMPRESSION"); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "Expected RELEASE"); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} +#endif /* WITH_ZSTD */ + + +/** + * @brief Test mixed scenario: Unsupported Magic → Success → CRC Error → Success + * + * Verifies parser correctly handles: + * - MessageSet 1: Unsupported MagicByte covering 3 messages (offsets 300-302) + * - MessageSet 2: 1 successful message (offset 303) + * - MessageSet 3: CRC error covering 2 messages (offsets 304-305) + * - MessageSet 4: 5 successful messages (offsets 306-310) + */ +static int unittest_msgset_mixed_magic_success_crc_success(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *buffer; + size_t buffer_size = 4096; + size_t offset = 0; + struct rd_kafka_toppar_ver tver = {.version = 11}; + const char *value1 = "x"; + size_t value_len1 = 1; + const char *values5[] = {"a", "b", "c", "d", "e"}; + size_t value_lens5[] = {1, 1, 1, 1, 1}; + int i; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = + ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-mixed3", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + buffer = rd_calloc(1, buffer_size); + + /* MessageSet 1: Unsupported MagicByte=99, 3 messages (offsets 300-302) + */ + ut_write_msgset_v2_header(buffer + offset, 300, 61 - 12, -1, + 99, /* Unsupported */ + 0, 0, 2, 0, 0, -1, -1, -1, 3); + offset += 12 + (61 - 12); + + /* MessageSet 2: Valid with 1 message (offset 303) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 303, 1, &value1, + &value_len1); + + /* MessageSet 3: CRC error, 2 messages (offsets 304-305) */ + ut_write_msgset_v2_header(buffer + offset, 304, 61 - 12, -1, 2, + 0xBADBAD, 0, 1, 0, 0, -1, -1, -1, 2); + offset += 12 + (61 - 12); + + /* MessageSet 4: Valid with 5 messages (offsets 306-310) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 306, 5, values5, + value_lens5); + + /* Parse all MessageSets */ + rkbuf = rd_kafka_buf_new_shadow(buffer, offset, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify MessageSet 1: 3 unsupported magic error ops (offsets 300-302) + */ + for (i = 0; i < 3; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected magic error op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, + "Expected NOT_IMPLEMENTED"); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 2: 1 successful FETCH op (offset 303) */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op"); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op"); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 303, + "Expected offset 303"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 3: 2 CRC error ops (offsets 304-305) */ + for (i = 0; i < 2; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected CRC error op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG"); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 4: 5 successful FETCH ops (offsets 306-310) */ + for (i = 0; i < 5; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op"); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 306 + i, + "Expected offset %d", 306 + i); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} + + +/** + * @brief Test all successful MessageSets + * + * Verifies parser correctly handles multiple consecutive successful + * MessageSets: + * - MessageSet 1: 2 messages (offsets 400-401) + * - MessageSet 2: 3 messages (offsets 402-404) + * - MessageSet 3: 1 message (offset 405) + * - MessageSet 4: 4 messages (offsets 406-409) + */ +static int unittest_msgset_all_success(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *buffer; + size_t buffer_size = 4096; + size_t offset = 0; + struct rd_kafka_toppar_ver tver = {.version = 11}; + const char *values2[] = {"m1", "m2"}; + size_t value_lens2[] = {2, 2}; + const char *values3[] = {"m3", "m4", "m5"}; + size_t value_lens3[] = {2, 2, 2}; + const char *value1 = "m6"; + size_t value_len1 = 2; + const char *values4[] = {"m7", "m8", "m9", "m10"}; + size_t value_lens4[] = {2, 2, 2, 3}; + int i; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = + ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-success", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + buffer = rd_calloc(1, buffer_size); + + /* Build 4 valid MessageSets */ + offset += ut_build_valid_msgset_v2(buffer + offset, 400, 2, values2, + value_lens2); + offset += ut_build_valid_msgset_v2(buffer + offset, 402, 3, values3, + value_lens3); + offset += ut_build_valid_msgset_v2(buffer + offset, 405, 1, &value1, + &value_len1); + offset += ut_build_valid_msgset_v2(buffer + offset, 406, 4, values4, + value_lens4); + + /* Parse all MessageSets */ + rkbuf = rd_kafka_buf_new_shadow(buffer, offset, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify all 10 messages received successfully (offsets 400-409) */ + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op %d, got timeout", + i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 400 + i, + "Expected offset %d, got %" PRId64, 400 + i, + rko->rko_u.fetch.rkm.rkm_offset); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} + + +/** + * @brief Test all error MessageSets in sequence + * + * Verifies parser correctly handles consecutive errors: + * - MessageSet 1: CRC error, 2 messages (offsets 500-501) + * - MessageSet 2: Unsupported MagicByte, 1 message (offset 502) + * - MessageSet 3: CRC error, 3 messages (offsets 503-505) + */ +static int unittest_msgset_all_errors(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *buffer; + size_t buffer_size = 4096; + size_t offset = 0; + struct rd_kafka_toppar_ver tver = {.version = 11}; + int i; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = + ut_create_mock_toppar(rkshare->rkshare_rk, "test-topic-errors", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + buffer = rd_calloc(1, buffer_size); + + /* MessageSet 1: CRC error, 2 messages (offsets 500-501) */ + ut_write_msgset_v2_header(buffer + offset, 500, 61 - 12, -1, 2, + 0xBADC001, 0, 1, 0, 0, -1, -1, -1, 2); + offset += 61; + + /* MessageSet 2: Unsupported MagicByte, 1 message (offset 502) */ + ut_write_msgset_v2_header(buffer + offset, 502, 61 - 12, -1, 99, 0, 0, + 0, 0, 0, -1, -1, -1, 1); + offset += 61; + + /* MessageSet 3: CRC error, 3 messages (offsets 503-505) */ + ut_write_msgset_v2_header(buffer + offset, 503, 61 - 12, -1, 2, + 0xBADC002, 0, 2, 0, 0, -1, -1, -1, 3); + offset += 61; + + RD_UT_SAY("Buffer size: %zu bytes", offset); + + /* Parse all MessageSets */ + rkbuf = rd_kafka_buf_new_shadow(buffer, offset, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + rd_kafka_resp_err_t parse_err = + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + RD_UT_SAY("Parse returned: %s", rd_kafka_err2str(parse_err)); + + /* Verify MessageSet 1: 2 CRC error ops (offsets 500-501) */ + for (i = 0; i < 2; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected CRC error op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 2: 1 unsupported magic error op (offset 502) */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected magic error op"); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, + "Expected NOT_IMPLEMENTED"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 3: 3 CRC error ops (offsets 503-505) */ + for (i = 0; i < 3; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected CRC error op %d", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR"); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG"); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} + + +#if WITH_ZSTD +/** + * @brief Test all 3 error types interleaved with valid messages + * + * Verifies parser continues through all error types: + * - MessageSet 1: CRC error (10 messages, offsets 0-9) + * - MessageSet 2: Valid (10 messages, offsets 10-19) + * - MessageSet 3: Decompression error (10 messages, offsets 20-29) + * - MessageSet 4: Valid (10 messages, offsets 30-39) + * - MessageSet 5: Unsupported MagicByte (10 messages, offsets 40-49) + * - MessageSet 6: Valid (10 messages, offsets 50-59) + * + * This is the critical test case: all batches must be processed, + * with valid messages delivered even after errors. + */ +static int unittest_msgset_all_error_types_with_valid(void) { + rd_kafka_share_t *rkshare; + rd_kafka_toppar_t *rktp; + rd_kafka_buf_t *rkbuf; + rd_kafka_q_t *response_q; + char *buffer; + size_t buffer_size = 8192; + size_t offset = 0; + struct rd_kafka_toppar_ver tver = {.version = 11}; + const char *values[] = {"a", "b", "c", "d", "e", + "f", "g", "h", "i", "j"}; + size_t value_lens[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + int i; + + RD_UT_BEGIN(); + + rkshare = ut_create_test_share_consumer(); + RD_UT_ASSERT(rkshare, "Failed to create share consumer"); + + rktp = ut_create_mock_toppar(rkshare->rkshare_rk, + "test-topic-all-errors", 0); + RD_UT_ASSERT(rktp, "Failed to create toppar"); + + response_q = rd_kafka_q_new(rkshare->rkshare_rk); + buffer = rd_calloc(1, buffer_size); + + /* MessageSet 1: CRC error with 10 messages (offsets 0-9) */ + ut_write_msgset_v2_header(buffer + offset, 0, 100, -1, 2, 0xBADC0001, 0, + 9, 0, 0, -1, -1, -1, 10); + offset += 112; + + /* MessageSet 2: Valid with 10 messages (offsets 10-19) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 10, 10, values, + value_lens); + + /* MessageSet 3: Decompression error with 10 messages (offsets 20-29) */ + char corrupted_data[32]; + memset(corrupted_data, 0xDE, sizeof(corrupted_data)); + size_t decomp_start = offset; + ut_write_msgset_v2_header(buffer + offset, 20, + 49 + sizeof(corrupted_data), -1, 2, 0, + 4, /* ZSTD compression */ + 9, 0, 0, -1, -1, -1, 10); + offset += 61; + memcpy(buffer + offset, corrupted_data, sizeof(corrupted_data)); + offset += sizeof(corrupted_data); + /* Fix CRC for decompression error (CRC valid, data corrupt) */ + uint32_t crc = rd_crc32c(0, buffer + decomp_start + 21, + 49 + sizeof(corrupted_data) - 4 - 1 - 4); + ut_put_be32(buffer + decomp_start + 17, crc); + + /* MessageSet 4: Valid with 10 messages (offsets 30-39) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 30, 10, values, + value_lens); + + /* MessageSet 5: Unsupported MagicByte with 10 messages (offsets 40-49) + */ + ut_write_msgset_v2_header(buffer + offset, 40, 100, -1, + 99, /* Bad magic */ + 0, 0, 9, 0, 0, -1, -1, -1, 10); + offset += 112; + + /* MessageSet 6: Valid with 10 messages (offsets 50-59) */ + offset += ut_build_valid_msgset_v2(buffer + offset, 50, 10, values, + value_lens); + + RD_UT_SAY("Total buffer size: %zu bytes, 6 MessageSets", offset); + + /* Parse all MessageSets */ + rkbuf = rd_kafka_buf_new_shadow(buffer, offset, rd_free); + rkbuf->rkbuf_rkb = rd_kafka_broker_internal(rkshare->rkshare_rk); + + rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver, response_q); + + /* Verify MessageSet 1: 10 CRC error ops (offsets 0-9) */ + RD_UT_SAY("Verifying MessageSet 1: CRC errors"); + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, + "Expected CRC error op %d of 10, got timeout", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_MSG, + "Expected BAD_MSG, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == 0 + i, + "Expected offset %d, got %" PRId64, i, + rko->rko_u.err.offset); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT ack_type for CRC error"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 2: 10 successful FETCH ops (offsets 10-19) */ + RD_UT_SAY("Verifying MessageSet 2: Valid messages"); + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected FETCH op %d of 10", i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 10 + i, + "Expected offset %d, got %" PRId64, 10 + i, + rko->rko_u.fetch.rkm.rkm_offset); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 3: 10 decompression error ops (offsets 20-29) */ + RD_UT_SAY("Verifying MessageSet 3: Decompression errors"); + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected decomp error op %d of 10", + i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__BAD_COMPRESSION, + "Expected BAD_COMPRESSION, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == 20 + i, + "Expected offset %d, got %" PRId64, 20 + i, + rko->rko_u.err.offset); + RD_UT_ASSERT( + rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE, + "Expected RELEASE ack_type for decompression error"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 4: 10 successful FETCH ops (offsets 30-39) */ + RD_UT_SAY("Verifying MessageSet 4: Valid messages after decomp error"); + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, + "Expected FETCH op %d of 10 after decomp error, " + "got timeout " + "(BUG: parser stopped!)", + i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 30 + i, + "Expected offset %d, got %" PRId64, 30 + i, + rko->rko_u.fetch.rkm.rkm_offset); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 5: 10 unsupported magic error ops (offsets 40-49) + */ + RD_UT_SAY("Verifying MessageSet 5: Unsupported MagicByte errors"); + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT(rko != NULL, "Expected magic error op %d of 10", + i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR, + "Expected CONSUMER_ERR, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_err == RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, + "Expected NOT_IMPLEMENTED, got %s", + rd_kafka_err2str(rko->rko_err)); + RD_UT_ASSERT(rko->rko_u.err.offset == 40 + i, + "Expected offset %d, got %" PRId64, 40 + i, + rko->rko_u.err.offset); + RD_UT_ASSERT(rko->rko_u.err.rkm.rkm_u.consumer.ack_type == + RD_KAFKA_SHARE_INTERNAL_ACK_REJECT, + "Expected REJECT ack_type for unsupported magic"); + rd_kafka_op_destroy(rko); + } + + /* Verify MessageSet 6: 10 successful FETCH ops (offsets 50-59) */ + RD_UT_SAY("Verifying MessageSet 6: Valid messages after magic error"); + for (i = 0; i < 10; i++) { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 1000, 0); + RD_UT_ASSERT( + rko != NULL, + "Expected FETCH op %d of 10 after magic error, got timeout " + "(BUG: parser stopped!)", + i); + RD_UT_ASSERT(rko->rko_type == RD_KAFKA_OP_FETCH, + "Expected FETCH op, got %d", rko->rko_type); + RD_UT_ASSERT(rko->rko_u.fetch.rkm.rkm_offset == 50 + i, + "Expected offset %d, got %" PRId64, 50 + i, + rko->rko_u.fetch.rkm.rkm_offset); + rd_kafka_op_destroy(rko); + } + + /* Verify no extra ops */ + { + rd_kafka_op_t *rko = rd_kafka_q_pop(response_q, 100, 0); + RD_UT_ASSERT(rko == NULL, "Unexpected extra op"); + } + + RD_UT_SAY( + "SUCCESS: All 6 MessageSets processed correctly (60 ops total)"); + + rd_kafka_buf_destroy(rkbuf); + rd_kafka_q_destroy_owner(response_q); + rd_kafka_toppar_destroy(rktp); + ut_destroy_share_consumer(rkshare); + + RD_UT_PASS(); +} +#endif /* WITH_ZSTD */ + + +/** + * @brief Run all MessageSet error handling unit tests + */ +int rd_kafka_unittest_msgset_errors(void) { + int fails = 0; + + /* Individual error type tests */ + fails += unittest_msgset_crc_error_share_consumer(); +#if WITH_ZSTD + fails += unittest_msgset_decompression_error_share_consumer(); +#endif + fails += unittest_msgset_unsupported_magic_share_consumer(); + + /* Mixed scenario tests */ + fails += unittest_msgset_mixed_success_crc_success(); +#if WITH_ZSTD + fails += unittest_msgset_mixed_crc_success_decomp(); +#endif + fails += unittest_msgset_mixed_magic_success_crc_success(); + fails += unittest_msgset_all_success(); + fails += unittest_msgset_all_errors(); + + /* Comprehensive test: all error types with valid messages */ +#if WITH_ZSTD + fails += unittest_msgset_all_error_types_with_valid(); +#endif + + return fails; +} diff --git a/lib/librdkafka-2.10.1/src/rdvarint.c b/lib/librdkafka-2.15.0/src/rdvarint.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdvarint.c rename to lib/librdkafka-2.15.0/src/rdvarint.c diff --git a/lib/librdkafka-2.10.1/src/rdvarint.h b/lib/librdkafka-2.15.0/src/rdvarint.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdvarint.h rename to lib/librdkafka-2.15.0/src/rdvarint.h diff --git a/lib/librdkafka-2.10.1/src/rdwin32.h b/lib/librdkafka-2.15.0/src/rdwin32.h similarity index 99% rename from lib/librdkafka-2.10.1/src/rdwin32.h rename to lib/librdkafka-2.15.0/src/rdwin32.h index 37c25843acb..40ea43a7ac0 100644 --- a/lib/librdkafka-2.10.1/src/rdwin32.h +++ b/lib/librdkafka-2.15.0/src/rdwin32.h @@ -42,7 +42,7 @@ #include /* for sockets + struct timeval */ #include #include - +#include /** * Types diff --git a/lib/librdkafka-2.10.1/src/rdxxhash.c b/lib/librdkafka-2.15.0/src/rdxxhash.c similarity index 100% rename from lib/librdkafka-2.10.1/src/rdxxhash.c rename to lib/librdkafka-2.15.0/src/rdxxhash.c diff --git a/lib/librdkafka-2.10.1/src/rdxxhash.h b/lib/librdkafka-2.15.0/src/rdxxhash.h similarity index 100% rename from lib/librdkafka-2.10.1/src/rdxxhash.h rename to lib/librdkafka-2.15.0/src/rdxxhash.h diff --git a/lib/librdkafka-2.10.1/src/regexp.c b/lib/librdkafka-2.15.0/src/regexp.c similarity index 100% rename from lib/librdkafka-2.10.1/src/regexp.c rename to lib/librdkafka-2.15.0/src/regexp.c diff --git a/lib/librdkafka-2.10.1/src/regexp.h b/lib/librdkafka-2.15.0/src/regexp.h similarity index 100% rename from lib/librdkafka-2.10.1/src/regexp.h rename to lib/librdkafka-2.15.0/src/regexp.h diff --git a/lib/librdkafka-2.10.1/src/snappy.c b/lib/librdkafka-2.15.0/src/snappy.c similarity index 100% rename from lib/librdkafka-2.10.1/src/snappy.c rename to lib/librdkafka-2.15.0/src/snappy.c diff --git a/lib/librdkafka-2.10.1/src/snappy.h b/lib/librdkafka-2.15.0/src/snappy.h similarity index 100% rename from lib/librdkafka-2.10.1/src/snappy.h rename to lib/librdkafka-2.15.0/src/snappy.h diff --git a/lib/librdkafka-2.10.1/src/snappy_compat.h b/lib/librdkafka-2.15.0/src/snappy_compat.h similarity index 100% rename from lib/librdkafka-2.10.1/src/snappy_compat.h rename to lib/librdkafka-2.15.0/src/snappy_compat.h diff --git a/lib/librdkafka-2.10.1/src/statistics_schema.json b/lib/librdkafka-2.15.0/src/statistics_schema.json similarity index 100% rename from lib/librdkafka-2.10.1/src/statistics_schema.json rename to lib/librdkafka-2.15.0/src/statistics_schema.json diff --git a/lib/librdkafka-2.10.1/src/tinycthread.c b/lib/librdkafka-2.15.0/src/tinycthread.c similarity index 100% rename from lib/librdkafka-2.10.1/src/tinycthread.c rename to lib/librdkafka-2.15.0/src/tinycthread.c diff --git a/lib/librdkafka-2.10.1/src/tinycthread.h b/lib/librdkafka-2.15.0/src/tinycthread.h similarity index 100% rename from lib/librdkafka-2.10.1/src/tinycthread.h rename to lib/librdkafka-2.15.0/src/tinycthread.h diff --git a/lib/librdkafka-2.10.1/src/tinycthread_extra.c b/lib/librdkafka-2.15.0/src/tinycthread_extra.c similarity index 94% rename from lib/librdkafka-2.10.1/src/tinycthread_extra.c rename to lib/librdkafka-2.15.0/src/tinycthread_extra.c index 6f6d0a59576..1c62cdce2fd 100644 --- a/lib/librdkafka-2.10.1/src/tinycthread_extra.c +++ b/lib/librdkafka-2.15.0/src/tinycthread_extra.c @@ -59,6 +59,17 @@ int thrd_is_current(thrd_t thr) { #endif } +unsigned long thrd_current_id(void) { +#ifdef _WIN32 + /* Windows makes a distinction between thread handle + * and thread id, which means we can't use the + * thrd_current() API that returns the handle. */ + return (unsigned long)GetCurrentThreadId(); +#else + return (unsigned long)(intptr_t)thrd_current(); +#endif +} + #ifdef _WIN32 void cnd_wait_enter(cnd_t *cond) { diff --git a/lib/librdkafka-2.10.1/src/tinycthread_extra.h b/lib/librdkafka-2.15.0/src/tinycthread_extra.h similarity index 98% rename from lib/librdkafka-2.10.1/src/tinycthread_extra.h rename to lib/librdkafka-2.15.0/src/tinycthread_extra.h index cb6b611ea7e..340f65ee8b3 100644 --- a/lib/librdkafka-2.10.1/src/tinycthread_extra.h +++ b/lib/librdkafka-2.15.0/src/tinycthread_extra.h @@ -55,6 +55,12 @@ int thrd_setname(const char *name); */ int thrd_is_current(thrd_t thr); +/** + * @brief Get current thread ID as an unsigned long. + * @return Current thread ID. + */ +unsigned long thrd_current_id(void); + #ifdef _WIN32 /** diff --git a/lib/librdkafka-2.10.1/src/win32_config.h b/lib/librdkafka-2.15.0/src/win32_config.h similarity index 100% rename from lib/librdkafka-2.10.1/src/win32_config.h rename to lib/librdkafka-2.15.0/src/win32_config.h diff --git a/lib/librdkafka-2.10.1/tests/.gitignore b/lib/librdkafka-2.15.0/tests/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/tests/.gitignore rename to lib/librdkafka-2.15.0/tests/.gitignore diff --git a/lib/librdkafka-2.10.1/tests/0000-unittests.c b/lib/librdkafka-2.15.0/tests/0000-unittests.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0000-unittests.c rename to lib/librdkafka-2.15.0/tests/0000-unittests.c diff --git a/lib/librdkafka-2.10.1/tests/0001-multiobj.c b/lib/librdkafka-2.15.0/tests/0001-multiobj.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0001-multiobj.c rename to lib/librdkafka-2.15.0/tests/0001-multiobj.c diff --git a/lib/librdkafka-2.10.1/tests/0002-unkpart.c b/lib/librdkafka-2.15.0/tests/0002-unkpart.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0002-unkpart.c rename to lib/librdkafka-2.15.0/tests/0002-unkpart.c diff --git a/lib/librdkafka-2.10.1/tests/0003-msgmaxsize.c b/lib/librdkafka-2.15.0/tests/0003-msgmaxsize.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0003-msgmaxsize.c rename to lib/librdkafka-2.15.0/tests/0003-msgmaxsize.c diff --git a/lib/librdkafka-2.10.1/tests/0004-conf.c b/lib/librdkafka-2.15.0/tests/0004-conf.c similarity index 92% rename from lib/librdkafka-2.10.1/tests/0004-conf.c rename to lib/librdkafka-2.15.0/tests/0004-conf.c index f733ccc4039..e129b707cc4 100644 --- a/lib/librdkafka-2.10.1/tests/0004-conf.c +++ b/lib/librdkafka-2.15.0/tests/0004-conf.c @@ -720,6 +720,75 @@ int main_0004_conf(int argc, char **argv) { rd_kafka_conf_destroy(conf); } +#if WITH_OAUTHBEARER_OIDC + { + TEST_SAY( + "Verify that https.ca.location is mutually " + "exclusive with https.ca.pem\n"); + + conf = rd_kafka_conf_new(); + + test_conf_set(conf, "https.ca.pem", + "-----BEGIN CERTIFICATE-----"); + test_conf_set(conf, "https.ca.location", + "/path/to/certificate.pem"); + + rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, + sizeof(errstr)); + TEST_ASSERT( + !rk, "Expected rd_kafka_new() to fail, but it succeeded"); + TEST_ASSERT(!strcmp(errstr, + "`https.ca.location` and " + "`https.ca.pem` are mutually exclusive"), + "Expected rd_kafka_new() to fail with: " + "\"`https.ca.location` and `https.ca.pem` " + "are mutually exclusive\", got: \"%s\"", + errstr); + rd_kafka_conf_destroy(conf); + } + { + TEST_SAY( + "Verify that https.ca.location gives an error when " + "set to an invalid path\n"); + + conf = rd_kafka_conf_new(); + + test_conf_set(conf, "https.ca.location", + "/?/does/!/not/exist!"); + + rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, + sizeof(errstr)); + TEST_ASSERT( + !rk, "Expected rd_kafka_new() to fail, but it succeeded"); + TEST_ASSERT(!strcmp(errstr, + "`https.ca.location` must be " + "an existing file or directory"), + "Expected rd_kafka_new() to fail with: " + "\"`https.ca.location` must be " + "an existing file or directory\", got: \"%s\"", + errstr); + rd_kafka_conf_destroy(conf); + } + { + TEST_SAY( + "Verify that https.ca.location doesn't give an error when " + "set to `probe`\n"); + + conf = rd_kafka_conf_new(); + + test_conf_set(conf, "https.ca.location", "probe"); + + rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, + sizeof(errstr)); + TEST_ASSERT( + rk, "Expected rd_kafka_new() not to fail, but it failed"); + + rd_kafka_destroy(rk); + } +#endif /* WITH_OAUTHBEARER_OIDC */ + + /* Verify that OpenSSL_AppLink is not needed on Windows (#3554) */ + #ifdef _WIN32 { FILE *fp; diff --git a/lib/librdkafka-2.10.1/tests/0005-order.c b/lib/librdkafka-2.15.0/tests/0005-order.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0005-order.c rename to lib/librdkafka-2.15.0/tests/0005-order.c diff --git a/lib/librdkafka-2.10.1/tests/0006-symbols.c b/lib/librdkafka-2.15.0/tests/0006-symbols.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0006-symbols.c rename to lib/librdkafka-2.15.0/tests/0006-symbols.c diff --git a/lib/librdkafka-2.10.1/tests/0007-autotopic.c b/lib/librdkafka-2.15.0/tests/0007-autotopic.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0007-autotopic.c rename to lib/librdkafka-2.15.0/tests/0007-autotopic.c diff --git a/lib/librdkafka-2.10.1/tests/0008-reqacks.c b/lib/librdkafka-2.15.0/tests/0008-reqacks.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0008-reqacks.c rename to lib/librdkafka-2.15.0/tests/0008-reqacks.c diff --git a/lib/librdkafka-2.15.0/tests/0009-mock_cluster.c b/lib/librdkafka-2.15.0/tests/0009-mock_cluster.c new file mode 100644 index 00000000000..1b092430816 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0009-mock_cluster.c @@ -0,0 +1,272 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2019-2022, Magnus Edenhill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + + +/** + * @name Verify that the builtin mock cluster works by producing to a topic + * and then consuming from it. + */ + + + +/** + * @brief Test rd_kafka_mock_topic_delete(). + * + * Create a topic, produce to it, delete it, then verify that + * a subsequent fetch receives UNKNOWN_TOPIC_OR_PARTITION. + */ +static void do_test_topic_delete(void) { + const char *topic = test_mk_topic_name("0009_topic_delete", 1); + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_t *p, *c; + rd_kafka_conf_t *conf; + rd_kafka_topic_t *rkt; + rd_kafka_resp_err_t err; + const char *bootstraps; + rd_kafka_topic_partition_list_t *parts; + rd_kafka_message_t *rkm; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(3, &bootstraps); + + /* Create topic explicitly so auto-create doesn't interfere + * after deletion. */ + TEST_CALL_ERR__(rd_kafka_mock_topic_create(mcluster, topic, 1, 1)); + + test_conf_init(&conf, NULL, 30); + + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "allow.auto.create.topics", "false"); + + /* Producer */ + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + p = test_create_handle(RD_KAFKA_PRODUCER, rd_kafka_conf_dup(conf)); + + rkt = test_create_producer_topic(p, topic, NULL); + + /* Produce */ + test_produce_msgs(p, rkt, 0, 0, 0, 10, NULL, 0); + rd_kafka_topic_destroy(rkt); + rd_kafka_destroy(p); + + /* Delete the topic */ + TEST_CALL_ERR__(rd_kafka_mock_topic_delete(mcluster, topic)); + + /* Verify deleting a non-existent topic returns error */ + err = rd_kafka_mock_topic_delete(mcluster, topic); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART, + "Expected UNKNOWN_TOPIC_OR_PART, got %s", + rd_kafka_err2str(err)); + + /* Consumer */ + test_conf_set(conf, "auto.offset.reset", "earliest"); + c = test_create_consumer(topic, NULL, conf, NULL); + + /* Assign */ + parts = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(parts, topic, 0); + test_consumer_assign("CONSUME_DELETED", c, parts); + rd_kafka_topic_partition_list_destroy(parts); + + /* Consume - expect no data messages since topic is deleted */ + rkm = rd_kafka_consumer_poll(c, 5000); + if (rkm) { + TEST_ASSERT(rkm != NULL && + rkm->err != RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected error from deleted topic"); + rd_kafka_message_destroy(rkm); + } + + rd_kafka_destroy(c); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test rd_kafka_mock_partition_delete_records(). + * + * Produce messages, delete records before an offset, then verify that: + * 1. Consuming from the beginning starts at the new start offset. + * 2. The API returns errors for invalid inputs. + */ +static void do_test_partition_delete_records(void) { + const char *topic = test_mk_topic_name("0009_delete_records", 1); + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_t *p, *c; + rd_kafka_conf_t *conf; + rd_kafka_topic_t *rkt; + rd_kafka_resp_err_t err; + const char *bootstraps; + const int msgcnt = 100; + const int64_t delete_before = 50; + rd_kafka_topic_partition_list_t *parts; + rd_kafka_message_t *rkm; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(3, &bootstraps); + + /* Create topic with 1 partition */ + TEST_CALL_ERR__(rd_kafka_mock_topic_create(mcluster, topic, 1, 1)); + + test_conf_init(&conf, NULL, 30); + + test_conf_set(conf, "bootstrap.servers", bootstraps); + + /* Producer */ + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + p = test_create_handle(RD_KAFKA_PRODUCER, rd_kafka_conf_dup(conf)); + + rkt = test_create_producer_topic(p, topic, NULL); + + /* Produce */ + test_produce_msgs(p, rkt, 0, 0, 0, msgcnt, NULL, 0); + rd_kafka_topic_destroy(rkt); + rd_kafka_destroy(p); + + /* Delete records before offset 50 */ + TEST_CALL_ERR__(rd_kafka_mock_partition_delete_records( + mcluster, topic, 0, delete_before)); + + /* Verify error for non-existent topic */ + err = rd_kafka_mock_partition_delete_records(mcluster, "no_such_topic", + 0, 10); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART, + "Expected UNKNOWN_TOPIC_OR_PART for bad topic, got %s", + rd_kafka_err2str(err)); + + /* Verify error for non-existent partition */ + err = rd_kafka_mock_partition_delete_records(mcluster, topic, 99, 10); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART, + "Expected UNKNOWN_TOPIC_OR_PART for bad partition, got %s", + rd_kafka_err2str(err)); + + /* Verify error for offset beyond end */ + err = rd_kafka_mock_partition_delete_records(mcluster, topic, 0, + msgcnt + 100); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE, + "Expected OFFSET_OUT_OF_RANGE, got %s", + rd_kafka_err2str(err)); + + /* Consumer */ + test_conf_set(conf, "auto.offset.reset", "earliest"); + c = test_create_consumer(topic, NULL, conf, NULL); + + /* Assign */ + parts = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(parts, topic, 0); + test_consumer_assign("CONSUME_AFTER_DELETE", c, parts); + rd_kafka_topic_partition_list_destroy(parts); + + /* Consume - first message should be at offset >= delete_before */ + rkm = rd_kafka_consumer_poll(c, 10000); + TEST_ASSERT(rkm != NULL, "Expected message, got NULL"); + TEST_ASSERT(rkm->err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected success, got %s", rd_kafka_err2str(rkm->err)); + TEST_ASSERT(rkm->offset >= delete_before, + "Expected first message offset >= %" PRId64 + ", got %" PRId64, + delete_before, rkm->offset); + TEST_SAY("First message after delete_records: offset %" PRId64 "\n", + rkm->offset); + rd_kafka_message_destroy(rkm); + + rd_kafka_destroy(c); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + + +int main_0009_mock_cluster(int argc, char **argv) { + const char *topic = test_mk_topic_name("0009_mock_cluster", 1); + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_t *p, *c; + rd_kafka_topic_t *rkt; + rd_kafka_conf_t *conf; + const int msgcnt = 100; + const char *bootstraps; + rd_kafka_topic_partition_list_t *parts; + + TEST_SKIP_MOCK_CLUSTER(0); + + mcluster = test_mock_cluster_new(3, &bootstraps); + + + test_conf_init(&conf, NULL, 30); + + test_conf_set(conf, "bootstrap.servers", bootstraps); + + /* Producer */ + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + p = test_create_handle(RD_KAFKA_PRODUCER, rd_kafka_conf_dup(conf)); + + /* Consumer */ + test_conf_set(conf, "auto.offset.reset", "earliest"); + c = test_create_consumer(topic, NULL, conf, NULL); + + rkt = test_create_producer_topic(p, topic, NULL); + test_wait_topic_exists(p, topic, 5000); + + /* Produce */ + test_produce_msgs(p, rkt, 0, RD_KAFKA_PARTITION_UA, 0, msgcnt, NULL, 0); + + /* Produce tiny messages */ + test_produce_msgs(p, rkt, 0, RD_KAFKA_PARTITION_UA, 0, msgcnt, "hello", + 5); + + rd_kafka_topic_destroy(rkt); + + /* Assign */ + parts = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(parts, topic, 0); + rd_kafka_topic_partition_list_add(parts, topic, 1); + rd_kafka_topic_partition_list_add(parts, topic, 2); + rd_kafka_topic_partition_list_add(parts, topic, 3); + test_consumer_assign("CONSUME", c, parts); + rd_kafka_topic_partition_list_destroy(parts); + + + /* Consume */ + test_consumer_poll("CONSUME", c, 0, -1, 0, msgcnt, NULL); + + rd_kafka_destroy(c); + rd_kafka_destroy(p); + + test_mock_cluster_destroy(mcluster); + + do_test_topic_delete(); + do_test_partition_delete_records(); + + return 0; +} diff --git a/lib/librdkafka-2.10.1/tests/0011-produce_batch.c b/lib/librdkafka-2.15.0/tests/0011-produce_batch.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0011-produce_batch.c rename to lib/librdkafka-2.15.0/tests/0011-produce_batch.c diff --git a/lib/librdkafka-2.10.1/tests/0012-produce_consume.c b/lib/librdkafka-2.15.0/tests/0012-produce_consume.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0012-produce_consume.c rename to lib/librdkafka-2.15.0/tests/0012-produce_consume.c diff --git a/lib/librdkafka-2.10.1/tests/0013-null-msgs.c b/lib/librdkafka-2.15.0/tests/0013-null-msgs.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0013-null-msgs.c rename to lib/librdkafka-2.15.0/tests/0013-null-msgs.c diff --git a/lib/librdkafka-2.10.1/tests/0014-reconsume-191.c b/lib/librdkafka-2.15.0/tests/0014-reconsume-191.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0014-reconsume-191.c rename to lib/librdkafka-2.15.0/tests/0014-reconsume-191.c diff --git a/lib/librdkafka-2.10.1/tests/0015-offset_seeks.c b/lib/librdkafka-2.15.0/tests/0015-offset_seeks.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0015-offset_seeks.c rename to lib/librdkafka-2.15.0/tests/0015-offset_seeks.c diff --git a/lib/librdkafka-2.10.1/tests/0016-client_swname.c b/lib/librdkafka-2.15.0/tests/0016-client_swname.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0016-client_swname.c rename to lib/librdkafka-2.15.0/tests/0016-client_swname.c diff --git a/lib/librdkafka-2.10.1/tests/0017-compression.c b/lib/librdkafka-2.15.0/tests/0017-compression.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0017-compression.c rename to lib/librdkafka-2.15.0/tests/0017-compression.c diff --git a/lib/librdkafka-2.10.1/tests/0018-cgrp_term.c b/lib/librdkafka-2.15.0/tests/0018-cgrp_term.c similarity index 75% rename from lib/librdkafka-2.10.1/tests/0018-cgrp_term.c rename to lib/librdkafka-2.15.0/tests/0018-cgrp_term.c index ed7c2754b03..b8d8e3160f9 100644 --- a/lib/librdkafka-2.10.1/tests/0018-cgrp_term.c +++ b/lib/librdkafka-2.15.0/tests/0018-cgrp_term.c @@ -330,9 +330,115 @@ static void do_test(rd_bool_t with_queue) { } +/** + * @name Test LeaveGroup handling when coordinator becomes unavailable + * + * Verifies that rd_kafka_cgrp_handle_LeaveGroup() correctly handles the case + * where the coordinator broker (rkb) is NULL when the consumer group initiates + * LeaveGroup during shutdown. + * + * Previously, rd_kafka_dbg() calls in the error path would dereference + * rkb->rkb_rk when rkb was NULL, causing a SIGSEGV crash. + * + * The fix uses the always-valid `rk` parameter instead of `rkb->rkb_rk`. + */ + +static int leavegroup_allowed_error; + +static int leavegroup_error_is_fatal_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + const char *reason) { + if (err == leavegroup_allowed_error || + err == RD_KAFKA_RESP_ERR__TRANSPORT || + err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN || + err == RD_KAFKA_RESP_ERR_GROUP_COORDINATOR_NOT_AVAILABLE || + err == RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP) { + TEST_SAY("Ignoring allowed error: %s: %s\n", + rd_kafka_err2name(err), reason); + return 0; + } + return 1; +} + +/** + * @brief Test that consumer close/destroy handles missing coordinator + * gracefully and does not crash. + * + * Scenario: + * 1. Create a consumer and subscribe to a topic + * 2. Wait for consumer to join group and establish coordinator connection + * 3. Make coordinator unavailable (broker goes down) + * 4. Close/destroy the consumer, which triggers LeaveGroup + * 5. Verify no crash occurs (the bug would cause SIGSEGV here) + */ +static void do_test_leavegroup_no_coordinator(void) { + rd_kafka_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_message_t *rkm; + + SUB_TEST(); + + TEST_SKIP_MOCK_CLUSTER(); + + test_curr->is_fatal_cb = leavegroup_error_is_fatal_cb; + leavegroup_allowed_error = RD_KAFKA_RESP_ERR__TRANSPORT; + + mcluster = test_mock_cluster_new(2, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 1, 2); + rd_kafka_mock_group_initial_rebalance_delay_ms(mcluster, 0); + rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1); + rd_kafka_mock_coordinator_set(mcluster, "group", topic, 1); + + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", topic); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "session.timeout.ms", "6000"); + test_conf_set(conf, "heartbeat.interval.ms", "1000"); + + consumer = test_create_consumer(topic, NULL, conf, NULL); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_subscribe(consumer, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + TEST_SAY("Waiting for consumer to join group and get assignment\n"); + rkm = rd_kafka_consumer_poll(consumer, 10000); + if (rkm) + rd_kafka_message_destroy(rkm); + + TEST_SAY( + "Simulating coordinator failure by making broker unavailable\n"); + rd_kafka_mock_broker_set_down(mcluster, 1); + + rd_sleep(1); + + TEST_SAY( + "Destroying consumer - this triggers LeaveGroup " + "with coordinator unavailable\n"); + rd_kafka_destroy(consumer); + + TEST_SAY("Consumer destroyed successfully (no crash)\n"); + + test_mock_cluster_destroy(mcluster); + + test_curr->is_fatal_cb = NULL; + leavegroup_allowed_error = RD_KAFKA_RESP_ERR_NO_ERROR; + + SUB_TEST_PASS(); +} + + int main_0018_cgrp_term(int argc, char **argv) { do_test(rd_false /* rd_kafka_consumer_close() */); do_test(rd_true /* rd_kafka_consumer_close_queue() */); + do_test_leavegroup_no_coordinator(); return 0; } diff --git a/lib/librdkafka-2.10.1/tests/0019-list_groups.c b/lib/librdkafka-2.15.0/tests/0019-list_groups.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0019-list_groups.c rename to lib/librdkafka-2.15.0/tests/0019-list_groups.c diff --git a/lib/librdkafka-2.10.1/tests/0020-destroy_hang.c b/lib/librdkafka-2.15.0/tests/0020-destroy_hang.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0020-destroy_hang.c rename to lib/librdkafka-2.15.0/tests/0020-destroy_hang.c diff --git a/lib/librdkafka-2.10.1/tests/0021-rkt_destroy.c b/lib/librdkafka-2.15.0/tests/0021-rkt_destroy.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0021-rkt_destroy.c rename to lib/librdkafka-2.15.0/tests/0021-rkt_destroy.c diff --git a/lib/librdkafka-2.10.1/tests/0022-consume_batch.c b/lib/librdkafka-2.15.0/tests/0022-consume_batch.c similarity index 82% rename from lib/librdkafka-2.10.1/tests/0022-consume_batch.c rename to lib/librdkafka-2.15.0/tests/0022-consume_batch.c index ab17ab92d6b..cf02456cb15 100644 --- a/lib/librdkafka-2.10.1/tests/0022-consume_batch.c +++ b/lib/librdkafka-2.15.0/tests/0022-consume_batch.c @@ -193,6 +193,57 @@ static void do_test_consume_batch_oauthbearer_cb(void) { rd_kafka_destroy(rk); } + + +static rd_bool_t share_refresh_called = rd_false; + +static void +share_refresh_cb(rd_kafka_t *rk, const char *oauthbearer_config, void *opaque) { + TEST_SAY("Share consumer refresh callback called\n"); + TEST_ASSERT(!share_refresh_called); + share_refresh_called = rd_true; + rd_kafka_oauthbearer_set_token_failure(rk, "Refresh called"); +} + +/** + * @brief Verify that the oauthbearer_refresh_cb() is triggered + * when using rd_kafka_share_poll() with a share consumer. + */ +static void do_test_share_poll_oauthbearer_cb(void) { + rd_kafka_share_t *rk; + rd_kafka_conf_t *conf; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + char errstr[512]; + + SUB_TEST_QUICK(); + + share_refresh_called = rd_false; + + conf = rd_kafka_conf_new(); + test_conf_set(conf, "security.protocol", "sasl_plaintext"); + test_conf_set(conf, "sasl.mechanism", "OAUTHBEARER"); + test_conf_set(conf, "group.id", "share-oauthbearer-cb-test"); + rd_kafka_conf_set_oauthbearer_token_refresh_cb(conf, share_refresh_cb); + + /* Create share consumer */ + rk = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rk, "Failed to create share consumer: %s", errstr); + + /* Poll to trigger the token refresh callback */ + err = rd_kafka_share_poll(rk, 1000, &batch); + if (err) + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + + TEST_SAY("share_refresh_called = %d\n", share_refresh_called); + TEST_ASSERT(share_refresh_called, + "Expected refresh callback to have been called " + "for share consumer"); + + rd_kafka_share_consumer_close(rk); + rd_kafka_share_destroy(rk); +} #endif @@ -259,7 +310,9 @@ static void do_test_consume_batch_non_existent_topic(void) { int main_0022_consume_batch(int argc, char **argv) { do_test_consume_batch(); - /* FIXME: this must be implemented in KIP-848 for compatibility. */ + /* Subscribing to a non-existent topic doesn't generate a + * UNKNOWN_TOPIC_OR_PART error with `consumer` group rebalance protocol. + */ if (test_consumer_group_protocol_classic()) { do_test_consume_batch_non_existent_topic(); } @@ -270,6 +323,7 @@ int main_0022_consume_batch(int argc, char **argv) { int main_0022_consume_batch_local(int argc, char **argv) { #if WITH_SASL_OAUTHBEARER do_test_consume_batch_oauthbearer_cb(); + do_test_share_poll_oauthbearer_cb(); #else TEST_SKIP("No OAUTHBEARER support\n"); #endif diff --git a/lib/librdkafka-2.10.1/tests/0025-timers.c b/lib/librdkafka-2.15.0/tests/0025-timers.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0025-timers.c rename to lib/librdkafka-2.15.0/tests/0025-timers.c diff --git a/lib/librdkafka-2.10.1/tests/0026-consume_pause.c b/lib/librdkafka-2.15.0/tests/0026-consume_pause.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0026-consume_pause.c rename to lib/librdkafka-2.15.0/tests/0026-consume_pause.c diff --git a/lib/librdkafka-2.10.1/tests/0028-long_topicnames.c b/lib/librdkafka-2.15.0/tests/0028-long_topicnames.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0028-long_topicnames.c rename to lib/librdkafka-2.15.0/tests/0028-long_topicnames.c diff --git a/lib/librdkafka-2.10.1/tests/0029-assign_offset.c b/lib/librdkafka-2.15.0/tests/0029-assign_offset.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0029-assign_offset.c rename to lib/librdkafka-2.15.0/tests/0029-assign_offset.c diff --git a/lib/librdkafka-2.10.1/tests/0030-offset_commit.c b/lib/librdkafka-2.15.0/tests/0030-offset_commit.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0030-offset_commit.c rename to lib/librdkafka-2.15.0/tests/0030-offset_commit.c diff --git a/lib/librdkafka-2.10.1/tests/0031-get_offsets.c b/lib/librdkafka-2.15.0/tests/0031-get_offsets.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0031-get_offsets.c rename to lib/librdkafka-2.15.0/tests/0031-get_offsets.c diff --git a/lib/librdkafka-2.10.1/tests/0033-regex_subscribe.c b/lib/librdkafka-2.15.0/tests/0033-regex_subscribe.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0033-regex_subscribe.c rename to lib/librdkafka-2.15.0/tests/0033-regex_subscribe.c diff --git a/lib/librdkafka-2.10.1/tests/0034-offset_reset.c b/lib/librdkafka-2.15.0/tests/0034-offset_reset.c similarity index 91% rename from lib/librdkafka-2.10.1/tests/0034-offset_reset.c rename to lib/librdkafka-2.15.0/tests/0034-offset_reset.c index 4a6a58f4dc1..5efcd358935 100644 --- a/lib/librdkafka-2.10.1/tests/0034-offset_reset.c +++ b/lib/librdkafka-2.15.0/tests/0034-offset_reset.c @@ -230,6 +230,7 @@ static void offset_reset_errors(void) { for (i = 0; i < (int)RD_ARRAYSIZE(test); i++) { rd_kafka_event_t *ev; rd_bool_t broker_down = rd_false; + rd_ts_t abs_timeout, now; /* Make sure consumer is connected */ test_wait_topic_exists(c, topic, 5000); @@ -259,13 +260,32 @@ static void offset_reset_errors(void) { test_consumer_assign_partition("ASSIGN", c, topic, partition, test[i].start_offset); + /* 5 seconds timeout */ + abs_timeout = test_clock() + 5 * 1000000; while (1) { + rd_bool_t not_auto_offset_reset_error; /* Poll until we see an AUTO_OFFSET_RESET error, * timeout, or a message, depending on what we're * looking for. */ - ev = rd_kafka_queue_poll(queue, 5000); + ev = rd_kafka_queue_poll(queue, 5000); + now = test_clock(); + not_auto_offset_reset_error = + ev && + rd_kafka_event_type(ev) == RD_KAFKA_EVENT_ERROR && + rd_kafka_event_error(ev) != + RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET; + + if (!ev || (not_auto_offset_reset_error && + now > abs_timeout)) { + if (ev) { + TEST_SAY( + "#%d: Ignoring %s event: %s\n", i, + rd_kafka_event_name(ev), + rd_kafka_event_error_string(ev)); + rd_kafka_event_destroy(ev); + ev = NULL; + } - if (!ev) { TEST_ASSERT(broker_down, "#%d: poll timeout, but broker " "was not down", @@ -278,12 +298,13 @@ static void offset_reset_errors(void) { broker_id); broker_down = rd_false; + /* 5 seconds timeout */ + abs_timeout = test_clock() + 5 * 1000000; } else if (rd_kafka_event_type(ev) == RD_KAFKA_EVENT_ERROR) { - if (rd_kafka_event_error(ev) != - RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET) { + if (not_auto_offset_reset_error) { TEST_SAY( "#%d: Ignoring %s event: %s\n", i, rd_kafka_event_name(ev), diff --git a/lib/librdkafka-2.10.1/tests/0035-api_version.c b/lib/librdkafka-2.15.0/tests/0035-api_version.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0035-api_version.c rename to lib/librdkafka-2.15.0/tests/0035-api_version.c diff --git a/lib/librdkafka-2.10.1/tests/0036-partial_fetch.c b/lib/librdkafka-2.15.0/tests/0036-partial_fetch.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0036-partial_fetch.c rename to lib/librdkafka-2.15.0/tests/0036-partial_fetch.c diff --git a/lib/librdkafka-2.10.1/tests/0037-destroy_hang_local.c b/lib/librdkafka-2.15.0/tests/0037-destroy_hang_local.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0037-destroy_hang_local.c rename to lib/librdkafka-2.15.0/tests/0037-destroy_hang_local.c diff --git a/lib/librdkafka-2.10.1/tests/0038-performance.c b/lib/librdkafka-2.15.0/tests/0038-performance.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0038-performance.c rename to lib/librdkafka-2.15.0/tests/0038-performance.c diff --git a/lib/librdkafka-2.10.1/tests/0039-event.c b/lib/librdkafka-2.15.0/tests/0039-event.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0039-event.c rename to lib/librdkafka-2.15.0/tests/0039-event.c diff --git a/lib/librdkafka-2.10.1/tests/0040-io_event.c b/lib/librdkafka-2.15.0/tests/0040-io_event.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0040-io_event.c rename to lib/librdkafka-2.15.0/tests/0040-io_event.c diff --git a/lib/librdkafka-2.10.1/tests/0041-fetch_max_bytes.c b/lib/librdkafka-2.15.0/tests/0041-fetch_max_bytes.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0041-fetch_max_bytes.c rename to lib/librdkafka-2.15.0/tests/0041-fetch_max_bytes.c diff --git a/lib/librdkafka-2.10.1/tests/0042-many_topics.c b/lib/librdkafka-2.15.0/tests/0042-many_topics.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0042-many_topics.c rename to lib/librdkafka-2.15.0/tests/0042-many_topics.c diff --git a/lib/librdkafka-2.10.1/tests/0043-no_connection.c b/lib/librdkafka-2.15.0/tests/0043-no_connection.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0043-no_connection.c rename to lib/librdkafka-2.15.0/tests/0043-no_connection.c diff --git a/lib/librdkafka-2.10.1/tests/0044-partition_cnt.c b/lib/librdkafka-2.15.0/tests/0044-partition_cnt.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0044-partition_cnt.c rename to lib/librdkafka-2.15.0/tests/0044-partition_cnt.c diff --git a/lib/librdkafka-2.10.1/tests/0045-subscribe_update.c b/lib/librdkafka-2.15.0/tests/0045-subscribe_update.c similarity index 87% rename from lib/librdkafka-2.10.1/tests/0045-subscribe_update.c rename to lib/librdkafka-2.15.0/tests/0045-subscribe_update.c index adf432b0623..6be18d1e0f6 100644 --- a/lib/librdkafka-2.10.1/tests/0045-subscribe_update.c +++ b/lib/librdkafka-2.15.0/tests/0045-subscribe_update.c @@ -806,6 +806,7 @@ static void do_test_resubscribe_with_regex() { /* Cleanup */ test_delete_topic(rk, topic1); test_delete_topic(rk, topic2); + test_delete_topic(rk, topic_a); test_consumer_close(rk); rd_kafka_queue_destroy(queue); @@ -814,11 +815,114 @@ static void do_test_resubscribe_with_regex() { rd_free(topic1); rd_free(topic2); + rd_free(topic_a); rd_free(group); SUB_TEST_PASS(); } +/** + * @brief Create many topics and apply several subscription + * updates, unsubscribing and re-subscribing too. + * After changing some subscriptions verifies that the assignment + * corresponds to last one. + * + * @param with_rebalance_cb Use a rebalance callback to perform the assignment. + * It needs to poll the consumer when awaiting for the + * assignment in this case. + */ +static void do_test_subscribe_many_updates(rd_bool_t with_rebalance_cb) { +#define TOPIC_CNT 100 + char *topics[TOPIC_CNT] = {0}; + char *topic; + char *group; + size_t i; + rd_kafka_t *rk; + rd_kafka_conf_t *conf; + const int partition_cnt = 4; + + SUB_TEST("%s", with_rebalance_cb ? "with rebalance callback" + : "without rebalance callback"); + + RD_ARRAY_FOREACH_INDEX(topic, topics, i) { + char topic_i[17]; + rd_snprintf(topic_i, sizeof(topic_i), "topic%" PRIusz, i); + topics[i] = rd_strdup(test_mk_topic_name(topic_i, 1)); + }; + group = topics[0]; + + test_conf_init(&conf, NULL, 60); + if (with_rebalance_cb) + rd_kafka_conf_set_rebalance_cb(conf, test_rebalance_cb); + rk = test_create_consumer(group, NULL, conf, NULL); + + TEST_SAY("Creating %d topics\n", TOPIC_CNT); + TEST_CALL_ERR__(test_CreateTopics_simple(rk, NULL, topics, TOPIC_CNT, + partition_cnt, NULL)); + test_wait_topic_exists(rk, topics[TOPIC_CNT - 1], 5000); + /* Give the cluster some more time to propagate metadata + * for TOPICS_CNT topics */ + rd_sleep(1); + + RD_ARRAY_FOREACH_INDEX(topic, topics, i) { + const int max_subscription_size = 5; + size_t j; + int k; + int subscription_size = + RD_MIN(max_subscription_size, TOPIC_CNT - i); + int expected_assignment_cnt = subscription_size * partition_cnt; + rd_kafka_topic_partition_list_t *expected_assignment = NULL; + + rd_kafka_topic_partition_list_t *subscription = + rd_kafka_topic_partition_list_new(subscription_size); + rd_bool_t check_expected_assignment = + (i % 5 == 0 || i == TOPIC_CNT - 1); + rd_bool_t do_unsubscribe = i % 7 == 0; + + if (check_expected_assignment) + expected_assignment = rd_kafka_topic_partition_list_new( + expected_assignment_cnt); + + for (j = i; j < i + subscription_size; j++) { + rd_kafka_topic_partition_list_add( + subscription, topics[j], RD_KAFKA_PARTITION_UA); + /* We unsubscribe every 7 iteration and + * we check assignments every 5 iteration so + * at 7 * 5 we unsubscribe and check that assignment + * is empty. */ + if (check_expected_assignment && !do_unsubscribe) + for (k = 0; k < partition_cnt; k++) + rd_kafka_topic_partition_list_add( + expected_assignment, topics[j], k); + } + TEST_CALL_ERR__(rd_kafka_subscribe(rk, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + if (do_unsubscribe) + TEST_CALL_ERR__(rd_kafka_unsubscribe(rk)); + if (check_expected_assignment) { + test_consumer_wait_assignment_topic_partition_list( + rk, + /* poll when we have a rebalance callback */ + with_rebalance_cb, expected_assignment, 30000); + rd_kafka_topic_partition_list_destroy( + expected_assignment); + } + }; + + TEST_CALL_ERR__( + test_DeleteTopics_simple(rk, NULL, topics, TOPIC_CNT, NULL)); + RD_ARRAY_FOREACH_INDEX(topic, topics, i) { + rd_free(topic); + }; + + test_consumer_close(rk); + rd_kafka_destroy(rk); + + SUB_TEST_PASS(); +#undef TOPIC_CNT +} + int main_0045_subscribe_update(int argc, char **argv) { if (!test_can_create_topics(1)) @@ -862,6 +966,12 @@ int main_0045_resubscribe_with_regex(int argc, char **argv) { return 0; } +int main_0045_subscribe_many_updates(int argc, char **argv) { + do_test_subscribe_many_updates(rd_false); + do_test_subscribe_many_updates(rd_true); + return 0; +} + int main_0045_subscribe_update_racks_mock(int argc, char **argv) { int use_replica_rack = 0; int use_client_rack = 0; diff --git a/lib/librdkafka-2.10.1/tests/0046-rkt_cache.c b/lib/librdkafka-2.15.0/tests/0046-rkt_cache.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0046-rkt_cache.c rename to lib/librdkafka-2.15.0/tests/0046-rkt_cache.c diff --git a/lib/librdkafka-2.10.1/tests/0047-partial_buf_tmout.c b/lib/librdkafka-2.15.0/tests/0047-partial_buf_tmout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0047-partial_buf_tmout.c rename to lib/librdkafka-2.15.0/tests/0047-partial_buf_tmout.c diff --git a/lib/librdkafka-2.10.1/tests/0048-partitioner.c b/lib/librdkafka-2.15.0/tests/0048-partitioner.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0048-partitioner.c rename to lib/librdkafka-2.15.0/tests/0048-partitioner.c diff --git a/lib/librdkafka-2.10.1/tests/0049-consume_conn_close.c b/lib/librdkafka-2.15.0/tests/0049-consume_conn_close.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0049-consume_conn_close.c rename to lib/librdkafka-2.15.0/tests/0049-consume_conn_close.c diff --git a/lib/librdkafka-2.10.1/tests/0050-subscribe_adds.c b/lib/librdkafka-2.15.0/tests/0050-subscribe_adds.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0050-subscribe_adds.c rename to lib/librdkafka-2.15.0/tests/0050-subscribe_adds.c diff --git a/lib/librdkafka-2.10.1/tests/0051-assign_adds.c b/lib/librdkafka-2.15.0/tests/0051-assign_adds.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0051-assign_adds.c rename to lib/librdkafka-2.15.0/tests/0051-assign_adds.c diff --git a/lib/librdkafka-2.10.1/tests/0052-msg_timestamps.c b/lib/librdkafka-2.15.0/tests/0052-msg_timestamps.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0052-msg_timestamps.c rename to lib/librdkafka-2.15.0/tests/0052-msg_timestamps.c diff --git a/lib/librdkafka-2.10.1/tests/0053-stats_cb.cpp b/lib/librdkafka-2.15.0/tests/0053-stats_cb.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0053-stats_cb.cpp rename to lib/librdkafka-2.15.0/tests/0053-stats_cb.cpp diff --git a/lib/librdkafka-2.10.1/tests/0054-offset_time.cpp b/lib/librdkafka-2.15.0/tests/0054-offset_time.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0054-offset_time.cpp rename to lib/librdkafka-2.15.0/tests/0054-offset_time.cpp diff --git a/lib/librdkafka-2.10.1/tests/0055-producer_latency.c b/lib/librdkafka-2.15.0/tests/0055-producer_latency.c similarity index 99% rename from lib/librdkafka-2.10.1/tests/0055-producer_latency.c rename to lib/librdkafka-2.15.0/tests/0055-producer_latency.c index 6cff6848b10..08ccf927b72 100644 --- a/lib/librdkafka-2.10.1/tests/0055-producer_latency.c +++ b/lib/librdkafka-2.15.0/tests/0055-producer_latency.c @@ -528,6 +528,8 @@ static void test_producer_latency_first_message(int case_number) { } int main_0055_producer_latency_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + int case_number; for (case_number = 0; case_number < 4; case_number++) { test_producer_latency_first_message(case_number); diff --git a/lib/librdkafka-2.10.1/tests/0056-balanced_group_mt.c b/lib/librdkafka-2.15.0/tests/0056-balanced_group_mt.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0056-balanced_group_mt.c rename to lib/librdkafka-2.15.0/tests/0056-balanced_group_mt.c diff --git a/lib/librdkafka-2.10.1/tests/0057-invalid_topic.cpp b/lib/librdkafka-2.15.0/tests/0057-invalid_topic.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0057-invalid_topic.cpp rename to lib/librdkafka-2.15.0/tests/0057-invalid_topic.cpp diff --git a/lib/librdkafka-2.10.1/tests/0058-log.cpp b/lib/librdkafka-2.15.0/tests/0058-log.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0058-log.cpp rename to lib/librdkafka-2.15.0/tests/0058-log.cpp diff --git a/lib/librdkafka-2.10.1/tests/0059-bsearch.cpp b/lib/librdkafka-2.15.0/tests/0059-bsearch.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0059-bsearch.cpp rename to lib/librdkafka-2.15.0/tests/0059-bsearch.cpp diff --git a/lib/librdkafka-2.10.1/tests/0060-op_prio.cpp b/lib/librdkafka-2.15.0/tests/0060-op_prio.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0060-op_prio.cpp rename to lib/librdkafka-2.15.0/tests/0060-op_prio.cpp diff --git a/lib/librdkafka-2.10.1/tests/0061-consumer_lag.cpp b/lib/librdkafka-2.15.0/tests/0061-consumer_lag.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0061-consumer_lag.cpp rename to lib/librdkafka-2.15.0/tests/0061-consumer_lag.cpp diff --git a/lib/librdkafka-2.10.1/tests/0062-stats_event.c b/lib/librdkafka-2.15.0/tests/0062-stats_event.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0062-stats_event.c rename to lib/librdkafka-2.15.0/tests/0062-stats_event.c diff --git a/lib/librdkafka-2.10.1/tests/0063-clusterid.cpp b/lib/librdkafka-2.15.0/tests/0063-clusterid.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0063-clusterid.cpp rename to lib/librdkafka-2.15.0/tests/0063-clusterid.cpp diff --git a/lib/librdkafka-2.10.1/tests/0064-interceptors.c b/lib/librdkafka-2.15.0/tests/0064-interceptors.c similarity index 90% rename from lib/librdkafka-2.10.1/tests/0064-interceptors.c rename to lib/librdkafka-2.15.0/tests/0064-interceptors.c index ddfb9e6bb4e..60792101538 100644 --- a/lib/librdkafka-2.10.1/tests/0064-interceptors.c +++ b/lib/librdkafka-2.15.0/tests/0064-interceptors.c @@ -468,6 +468,52 @@ static void do_test_conf_copy(const char *topic) { } +/** + * @brief Test for Issue #4142: Segfault when on_conf_destroy callback + * is registered and rd_kafka_new() fails. + * + * The bug was in the error handling path of rd_kafka_new() where interceptors + * were being destroyed twice, causing a segfault when the application + * properly called rd_kafka_conf_destroy() after rd_kafka_new() failure. + */ +static rd_kafka_resp_err_t on_conf_destroy_failed_new(void *ic_opaque) { + int *call_cnt = (int *)ic_opaque; + (*call_cnt)++; + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static void do_test_conf_destroy_interceptor_with_failed_new(void) { + rd_kafka_conf_t *conf; + rd_kafka_t *rk; + char errstr[512]; + int call_cnt = 0; + + TEST_SAY(_C_MAG "[ %s ]\n" _C_CLR, __FUNCTION__); + + conf = rd_kafka_conf_new(); + + TEST_CALL_ERR__(rd_kafka_conf_interceptor_add_on_conf_destroy( + conf, "on_conf_destroy_failed_new", on_conf_destroy_failed_new, + &call_cnt)); + + /* Set invalid SSL configuration to cause rd_kafka_new() to fail */ + test_conf_set(conf, "security.protocol", "ssl"); + test_conf_set(conf, "ssl.ca.location", "/dev/null"); + + /* This should fail due to invalid SSL config */ + rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); + TEST_ASSERT( + !rk, "rd_kafka_new() should have failed with invalid SSL config"); + + rd_kafka_conf_destroy(conf); + + /* Verify on_conf_destroy was called exactly once */ + TEST_ASSERT(call_cnt == 1, + "on_conf_destroy should have been called once, got %d", + call_cnt); +} + + int main_0064_interceptors(int argc, char **argv) { const char *topic = test_mk_topic_name(__FUNCTION__, 1); @@ -477,5 +523,7 @@ int main_0064_interceptors(int argc, char **argv) { do_test_conf_copy(topic); + do_test_conf_destroy_interceptor_with_failed_new(); + return 0; } diff --git a/lib/librdkafka-2.10.1/tests/0065-yield.cpp b/lib/librdkafka-2.15.0/tests/0065-yield.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0065-yield.cpp rename to lib/librdkafka-2.15.0/tests/0065-yield.cpp diff --git a/lib/librdkafka-2.10.1/tests/0066-plugins.cpp b/lib/librdkafka-2.15.0/tests/0066-plugins.cpp similarity index 89% rename from lib/librdkafka-2.10.1/tests/0066-plugins.cpp rename to lib/librdkafka-2.15.0/tests/0066-plugins.cpp index 7b5e7b00fb9..e5705354e5e 100644 --- a/lib/librdkafka-2.10.1/tests/0066-plugins.cpp +++ b/lib/librdkafka-2.15.0/tests/0066-plugins.cpp @@ -69,6 +69,20 @@ static void do_test_plugin() { NULL, }; + RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); + if (conf->set("plugin.library.paths", + "interceptor_test/interceptor_test_dummy", errstr) && + errstr.find( + "Configuration property \"plugin.library.paths\" not supported") != + std::string::npos) { + delete conf; + Test::Skip( + "Configuration property \"plugin.library.paths\" not supported in this " + "build\n"); + return; + } + delete conf; + char cwd[512], *pcwd; #ifdef _WIN32 pcwd = _getcwd(cwd, sizeof(cwd) - 1); @@ -84,7 +98,7 @@ static void do_test_plugin() { ictest_cnt_init(&ictest.on_new, 1, 1); /* Config for intercepted client */ - RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); + conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL); for (int i = 0; config[i]; i += 2) { Test::Say(tostr() << "set(" << config[i] << ", " << config[i + 1] << ")\n"); diff --git a/lib/librdkafka-2.10.1/tests/0067-empty_topic.cpp b/lib/librdkafka-2.15.0/tests/0067-empty_topic.cpp similarity index 94% rename from lib/librdkafka-2.10.1/tests/0067-empty_topic.cpp rename to lib/librdkafka-2.15.0/tests/0067-empty_topic.cpp index 2db9ee87350..c2a1c392779 100644 --- a/lib/librdkafka-2.10.1/tests/0067-empty_topic.cpp +++ b/lib/librdkafka-2.15.0/tests/0067-empty_topic.cpp @@ -70,6 +70,9 @@ static void do_test_empty_topic_consumer() { RdKafka::err2str(err)); delete md; + /* Await propagation to the whole cluster */ + test_wait_topic_exists(consumer->c_ptr(), topic.c_str(), 1 * 1000); + /* Start consumer */ err = consumer->start(rkt, partition, RdKafka::Topic::OFFSET_BEGINNING); if (err) diff --git a/lib/librdkafka-2.10.1/tests/0068-produce_timeout.c b/lib/librdkafka-2.15.0/tests/0068-produce_timeout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0068-produce_timeout.c rename to lib/librdkafka-2.15.0/tests/0068-produce_timeout.c diff --git a/lib/librdkafka-2.10.1/tests/0069-consumer_add_parts.c b/lib/librdkafka-2.15.0/tests/0069-consumer_add_parts.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0069-consumer_add_parts.c rename to lib/librdkafka-2.15.0/tests/0069-consumer_add_parts.c diff --git a/lib/librdkafka-2.10.1/tests/0070-null_empty.cpp b/lib/librdkafka-2.15.0/tests/0070-null_empty.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0070-null_empty.cpp rename to lib/librdkafka-2.15.0/tests/0070-null_empty.cpp diff --git a/lib/librdkafka-2.10.1/tests/0072-headers_ut.c b/lib/librdkafka-2.15.0/tests/0072-headers_ut.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0072-headers_ut.c rename to lib/librdkafka-2.15.0/tests/0072-headers_ut.c diff --git a/lib/librdkafka-2.10.1/tests/0073-headers.c b/lib/librdkafka-2.15.0/tests/0073-headers.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0073-headers.c rename to lib/librdkafka-2.15.0/tests/0073-headers.c diff --git a/lib/librdkafka-2.10.1/tests/0074-producev.c b/lib/librdkafka-2.15.0/tests/0074-producev.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0074-producev.c rename to lib/librdkafka-2.15.0/tests/0074-producev.c diff --git a/lib/librdkafka-2.10.1/tests/0075-retry.c b/lib/librdkafka-2.15.0/tests/0075-retry.c similarity index 90% rename from lib/librdkafka-2.10.1/tests/0075-retry.c rename to lib/librdkafka-2.15.0/tests/0075-retry.c index 5679f9fe012..958e0b70787 100644 --- a/lib/librdkafka-2.10.1/tests/0075-retry.c +++ b/lib/librdkafka-2.15.0/tests/0075-retry.c @@ -60,6 +60,8 @@ static struct { int term; /* Number of created sockets */ int num_sockets; + /* Current test, available to all threads as it's not TLS */ + struct test *test_curr; } ctrl; static int ctrl_thrd_main(void *arg) { @@ -70,6 +72,10 @@ static int ctrl_thrd_main(void *arg) { int64_t now; cnd_timedwait_ms(&ctrl.cnd, &ctrl.lock, 10); + if (!test_socket_find(ctrl.test_curr, ctrl.skm)) + /* Socket was closed and destroyed when + * releasing the lock. */ + break; if (ctrl.cmd.ts_at) { ctrl.next.ts_at = ctrl.cmd.ts_at; @@ -181,14 +187,26 @@ static void do_test_low_socket_timeout(const char *topic) { mtx_init(&ctrl.lock, mtx_plain); cnd_init(&ctrl.cnd); + ctrl.test_curr = test_curr; TEST_SAY("Test Metadata request retries on timeout\n"); + test_create_topic_wait_exists(NULL, topic, 1, -1, 5000); + test_conf_init(&conf, NULL, 60); test_conf_set(conf, "socket.timeout.ms", "1000"); test_conf_set(conf, "socket.max.fails", "12345"); test_conf_set(conf, "retry.backoff.ms", "5000"); test_conf_set(conf, "retry.backoff.max.ms", "5000"); + /* Sparse re-connect interval depends on `reconnect.backoff.ms` / 2 + * increasing it allows to saturate the sparse re-connect interval + * and avoid to concurrent connects: the first for setting the bootstrap + * brokers and the second for adding a new topic to get metadata for. + * This way the test is more reliable especially when using valgrind + * as it needs to keep the last connection only, + * to add a delay later. */ + test_conf_set(conf, "reconnect.backoff.ms", "2000"); + /* Avoid api version requests (with their own timeout) to get in * the way of our test */ test_conf_set(conf, "api.version.request", "false"); @@ -198,7 +216,6 @@ static void do_test_low_socket_timeout(const char *topic) { rk = test_create_handle(RD_KAFKA_PRODUCER, conf); rkt = test_create_producer_topic(rk, topic, NULL); - test_wait_topic_exists(rk, topic, 5000); TEST_SAY("Waiting for sockem connect..\n"); mtx_lock(&ctrl.lock); diff --git a/lib/librdkafka-2.10.1/tests/0076-produce_retry.c b/lib/librdkafka-2.15.0/tests/0076-produce_retry.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0076-produce_retry.c rename to lib/librdkafka-2.15.0/tests/0076-produce_retry.c diff --git a/lib/librdkafka-2.10.1/tests/0077-compaction.c b/lib/librdkafka-2.15.0/tests/0077-compaction.c similarity index 95% rename from lib/librdkafka-2.10.1/tests/0077-compaction.c rename to lib/librdkafka-2.15.0/tests/0077-compaction.c index 433c249b008..76876724fc9 100644 --- a/lib/librdkafka-2.10.1/tests/0077-compaction.c +++ b/lib/librdkafka-2.15.0/tests/0077-compaction.c @@ -187,8 +187,8 @@ static void do_test_compaction(int msgs_per_key, const char *compression) { "--partitions %d " "--replication-factor 1 " "--config cleanup.policy=compact " - "--config segment.ms=10000 " - "--config segment.bytes=10000 " + "--config segment.ms=500 " + "--config segment.bytes=1048576 " "--config min.cleanable.dirty.ratio=0.01 " "--config delete.retention.ms=86400 " "--config file.delete.delay.ms=10000 " @@ -209,8 +209,10 @@ static void do_test_compaction(int msgs_per_key, const char *compression) { /* The low watermark is not updated on message deletion(compaction) * but on segment deletion, so fill up the first segment with - * random messages eligible for hasty compaction. */ - produce_compactable_msgs(topic, 0, partition, fillcnt, 1000); + * random messages eligible for hasty compaction. + * Use 60KB messages so that 20 of them (~1.2MB) exceed + * segment.bytes (1MB min in Kafka 4.1+) and force segment rolls. */ + produce_compactable_msgs(topic, 0, partition, fillcnt, 60000); /* Populate a correct msgver for later comparison after compact. */ test_msgver_init(&mv_correct, testid); @@ -290,13 +292,14 @@ static void do_test_compaction(int msgs_per_key, const char *compression) { msgcounter = cnt; test_wait_delivery(rk, &msgcounter); - /* Trigger compaction by filling up the segment with dummy messages, + /* Trigger compaction by filling up segments with dummy messages, * do it in chunks to avoid too good compression which then won't * fill up the segments.. * We can't reuse the existing producer instance because it * might be using compression which makes it hard to know how - * much data we need to produce to trigger compaction. */ - produce_compactable_msgs(topic, 0, partition, 20, 1024); + * much data we need to produce to trigger compaction. + * Use 60KB messages to exceed segment.bytes (1MB min in 4.1+). */ + produce_compactable_msgs(topic, 0, partition, 20, 60000); /* Wait for compaction: * this doesn't really work because the low watermark offset diff --git a/lib/librdkafka-2.10.1/tests/0078-c_from_cpp.cpp b/lib/librdkafka-2.15.0/tests/0078-c_from_cpp.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0078-c_from_cpp.cpp rename to lib/librdkafka-2.15.0/tests/0078-c_from_cpp.cpp diff --git a/lib/librdkafka-2.10.1/tests/0079-fork.c b/lib/librdkafka-2.15.0/tests/0079-fork.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0079-fork.c rename to lib/librdkafka-2.15.0/tests/0079-fork.c diff --git a/lib/librdkafka-2.10.1/tests/0080-admin_ut.c b/lib/librdkafka-2.15.0/tests/0080-admin_ut.c similarity index 98% rename from lib/librdkafka-2.10.1/tests/0080-admin_ut.c rename to lib/librdkafka-2.15.0/tests/0080-admin_ut.c index 55067bf9734..a9f0e1181fe 100644 --- a/lib/librdkafka-2.10.1/tests/0080-admin_ut.c +++ b/lib/librdkafka-2.15.0/tests/0080-admin_ut.c @@ -528,22 +528,44 @@ static void do_test_ListConsumerGroups(const char *what, q = useq ? useq : rd_kafka_queue_new(rk); if (with_options) { - rd_kafka_consumer_group_state_t duplicate[2] = { + rd_kafka_error_t *error; + rd_kafka_consumer_group_state_t duplicate_states[2] = { RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY, RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY}; + rd_kafka_consumer_group_type_t duplicate_types[2] = { + RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC, + RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC}; + rd_kafka_consumer_group_type_t unknown_type[1] = { + RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN}; options = rd_kafka_AdminOptions_new( rk, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS); /* Test duplicate error on match states */ - rd_kafka_error_t *error = - rd_kafka_AdminOptions_set_match_consumer_group_states( - options, duplicate, 2); + error = rd_kafka_AdminOptions_set_match_consumer_group_states( + options, duplicate_states, 2); TEST_ASSERT(error && rd_kafka_error_code(error), "%s", "Expected error on duplicate states," " got no error"); rd_kafka_error_destroy(error); + /* Test duplicate error on match group types */ + error = rd_kafka_AdminOptions_set_match_consumer_group_types( + options, duplicate_types, 2); + TEST_ASSERT(error && rd_kafka_error_code(error), "%s", + "Expected error on duplicate group types," + " got no error"); + rd_kafka_error_destroy(error); + + /* Test invalid args error on setting UNKNOWN group type in + * match group types */ + error = rd_kafka_AdminOptions_set_match_consumer_group_types( + options, unknown_type, 1); + TEST_ASSERT(error && rd_kafka_error_code(error), "%s", + "Expected error on Unknown group type," + " got no error"); + rd_kafka_error_destroy(error); + exp_timeout = MY_SOCKET_TIMEOUT_MS * 2; TEST_CALL_ERR__(rd_kafka_AdminOptions_set_request_timeout( options, exp_timeout, errstr, sizeof(errstr))); diff --git a/lib/librdkafka-2.10.1/tests/0081-admin.c b/lib/librdkafka-2.15.0/tests/0081-admin.c similarity index 97% rename from lib/librdkafka-2.10.1/tests/0081-admin.c rename to lib/librdkafka-2.15.0/tests/0081-admin.c index c8c2a3a0a05..f16f958e586 100644 --- a/lib/librdkafka-2.10.1/tests/0081-admin.c +++ b/lib/librdkafka-2.15.0/tests/0081-admin.c @@ -2823,135 +2823,63 @@ static void do_test_DeleteGroups(const char *what, } /** - * @brief Test list groups, creating consumers for a set of groups, - * listing and deleting them at the end. + * @brief Helper for do_test_ListConsumerGroups, makes ListConsumerGroups call + * and checks returned group count equals to \p exp_found. + * + * Parameter \p exp_type, if not UNKNOWN, needs to match + * returned groups type. + * + * If \p match_states is true, then the state of the + * returned groups is checked and expected to be `EMPTY`. */ -static void do_test_ListConsumerGroups(const char *what, - rd_kafka_t *rk, - rd_kafka_queue_t *useq, - int request_timeout, - rd_bool_t match_states) { -#define TEST_LIST_CONSUMER_GROUPS_CNT 4 - rd_kafka_queue_t *q; - rd_kafka_AdminOptions_t *options = NULL; - rd_kafka_event_t *rkev = NULL; - rd_kafka_resp_err_t err; - size_t valid_cnt, error_cnt; - rd_bool_t is_simple_consumer_group; - rd_kafka_consumer_group_state_t state; - char errstr[512]; - const char *errstr2, *group_id; - char *list_consumer_groups[TEST_LIST_CONSUMER_GROUPS_CNT]; - const int partitions_cnt = 1; - const int msgs_cnt = 100; - size_t i, found; - char *topic; - rd_kafka_metadata_topic_t exp_mdtopic = {0}; - int64_t testid = test_id_generate(); - test_timing_t timing; - rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR; - const rd_kafka_ListConsumerGroups_result_t *res; - const rd_kafka_ConsumerGroupListing_t **groups; - rd_bool_t has_match_states = - test_broker_version >= TEST_BRKVER(2, 7, 0, 0); - - SUB_TEST_QUICK( - "%s ListConsumerGroups with %s, request_timeout %d" - ", match_states %s", - rd_kafka_name(rk), what, request_timeout, RD_STR_ToF(match_states)); - - q = useq ? useq : rd_kafka_queue_new(rk); - - if (request_timeout != -1) { - options = rd_kafka_AdminOptions_new( - rk, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS); - - if (match_states) { - rd_kafka_consumer_group_state_t empty = - RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY; - - TEST_CALL_ERROR__( - rd_kafka_AdminOptions_set_match_consumer_group_states( - options, &empty, 1)); - } - - TEST_CALL_ERR__(rd_kafka_AdminOptions_set_request_timeout( - options, request_timeout, errstr, sizeof(errstr))); - } - - - topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1)); - exp_mdtopic.topic = topic; - - /* Create the topics first. */ - test_CreateTopics_simple(rk, NULL, &topic, 1, partitions_cnt, NULL); - - /* Verify that topics are reported by metadata */ - test_wait_metadata_update(rk, &exp_mdtopic, 1, NULL, 0, 15 * 1000); - - /* Produce 100 msgs */ - test_produce_msgs_easy(topic, testid, 0, msgs_cnt); - - for (i = 0; i < TEST_LIST_CONSUMER_GROUPS_CNT; i++) { - char *group = rd_strdup(test_mk_topic_name(__FUNCTION__, 1)); - test_consume_msgs_easy(group, topic, testid, -1, msgs_cnt, - NULL); - list_consumer_groups[i] = group; - } +static void +test_ListConsumerGroups_helper(rd_kafka_t *rk, + rd_kafka_AdminOptions_t *option, + rd_kafka_queue_t *q, + char **list_consumer_groups, + size_t list_consumer_groups_cnt, + size_t exp_found, + rd_kafka_consumer_group_type_t exp_type, + rd_bool_t match_states) { + rd_kafka_event_t *rkev = NULL; + const rd_kafka_ListConsumerGroups_result_t *result = NULL; + size_t group_cnt; + size_t error_cnt; + size_t found; + size_t i, j; + const rd_kafka_ConsumerGroupListing_t **groups = NULL; - TIMING_START(&timing, "ListConsumerGroups"); TEST_SAY("Call ListConsumerGroups\n"); - rd_kafka_ListConsumerGroups(rk, options, q); - TIMING_ASSERT_LATER(&timing, 0, 50); - - TIMING_START(&timing, "ListConsumerGroups.queue_poll"); - + rd_kafka_ListConsumerGroups(rk, option, q); /* Poll result queue for ListConsumerGroups result. * Print but otherwise ignore other event types * (typically generic Error events). */ while (1) { rkev = rd_kafka_queue_poll(q, tmout_multip(20 * 1000)); - TEST_SAY("ListConsumerGroups: got %s in %.3fms\n", - rd_kafka_event_name(rkev), - TIMING_DURATION(&timing) / 1000.0f); if (rkev == NULL) continue; + if (rd_kafka_event_error(rkev)) TEST_SAY("%s: %s\n", rd_kafka_event_name(rkev), rd_kafka_event_error_string(rkev)); if (rd_kafka_event_type(rkev) == - RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT) { + RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT) break; - } rd_kafka_event_destroy(rkev); } /* Convert event to proper result */ - res = rd_kafka_event_ListConsumerGroups_result(rkev); - TEST_ASSERT(res, "expected ListConsumerGroups_result, got %s", + result = rd_kafka_event_ListConsumerGroups_result(rkev); + TEST_ASSERT(result, "expected ListConsumerGroups_result, got %s", rd_kafka_event_name(rkev)); - /* Expecting error */ - err = rd_kafka_event_error(rkev); - errstr2 = rd_kafka_event_error_string(rkev); - TEST_ASSERT(err == exp_err, - "expected ListConsumerGroups to return %s, got %s (%s)", - rd_kafka_err2str(exp_err), rd_kafka_err2str(err), - err ? errstr2 : "n/a"); + if (rd_kafka_event_error(rkev)) + TEST_FAIL("ListConsumerGroups failed with %s", + rd_kafka_event_error_string(rkev)); - TEST_SAY("ListConsumerGroups: returned %s (%s)\n", - rd_kafka_err2str(err), err ? errstr2 : "n/a"); - - groups = rd_kafka_ListConsumerGroups_result_valid(res, &valid_cnt); - rd_kafka_ListConsumerGroups_result_errors(res, &error_cnt); - - /* Other tests could be running */ - TEST_ASSERT(valid_cnt >= TEST_LIST_CONSUMER_GROUPS_CNT, - "expected ListConsumerGroups to return at least %" PRId32 - " valid groups," - " got %zu", - TEST_LIST_CONSUMER_GROUPS_CNT, valid_cnt); + groups = rd_kafka_ListConsumerGroups_result_valid(result, &group_cnt); + rd_kafka_ListConsumerGroups_result_errors(result, &error_cnt); TEST_ASSERT(error_cnt == 0, "expected ListConsumerGroups to return 0 errors," @@ -2959,23 +2887,32 @@ static void do_test_ListConsumerGroups(const char *what, error_cnt); found = 0; - for (i = 0; i < valid_cnt; i++) { - int j; - const rd_kafka_ConsumerGroupListing_t *group; - group = groups[i]; - group_id = rd_kafka_ConsumerGroupListing_group_id(group); - is_simple_consumer_group = + for (i = 0; i < group_cnt; i++) { + const rd_kafka_ConsumerGroupListing_t *group = groups[i]; + const char *group_id = + rd_kafka_ConsumerGroupListing_group_id(group); + int is_simple_consumer_group = rd_kafka_ConsumerGroupListing_is_simple_consumer_group( group); - state = rd_kafka_ConsumerGroupListing_state(group); - for (j = 0; j < TEST_LIST_CONSUMER_GROUPS_CNT; j++) { + rd_kafka_consumer_group_state_t state = + rd_kafka_ConsumerGroupListing_state(group); + rd_kafka_consumer_group_type_t type = + rd_kafka_ConsumerGroupListing_type(group); + for (j = 0; j < list_consumer_groups_cnt; j++) { if (!strcmp(list_consumer_groups[j], group_id)) { found++; TEST_ASSERT(!is_simple_consumer_group, "expected a normal group," " got a simple group"); + if (exp_type != + RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN) { + TEST_ASSERT( + type == exp_type, + "Expected the Consumer Group Type " + "to be set by the Broker."); + } - if (!has_match_states) + if (!match_states) break; TEST_ASSERT( @@ -2988,13 +2925,137 @@ static void do_test_ListConsumerGroups(const char *what, } } } - TEST_ASSERT(found == TEST_LIST_CONSUMER_GROUPS_CNT, - "expected to find %d" - " started groups," + + TEST_ASSERT(found == exp_found, + "expected to find %" PRIusz + " consumer groups," " got %" PRIusz, - TEST_LIST_CONSUMER_GROUPS_CNT, found); + exp_found, found); rd_kafka_event_destroy(rkev); +} + +/** + * @brief Test list groups, creating consumers for a set of groups, + * listing and deleting them at the end. + */ +static void do_test_ListConsumerGroups(const char *what, + rd_kafka_t *rk, + rd_kafka_queue_t *useq, + int request_timeout, + rd_bool_t match_states, + rd_bool_t match_types) { +#define TEST_LIST_CONSUMER_GROUPS_CNT 4 + rd_kafka_queue_t *q; + rd_kafka_AdminOptions_t *options; + char errstr[512]; + char *list_consumer_groups[TEST_LIST_CONSUMER_GROUPS_CNT]; + const int partitions_cnt = 1; + const int msgs_cnt = 100; + char *topic; + size_t i; + rd_kafka_metadata_topic_t exp_mdtopic = {0}; + int64_t testid = test_id_generate(); + rd_bool_t has_match_states = + test_broker_version >= TEST_BRKVER(2, 7, 0, 0); + rd_bool_t has_match_types = + test_broker_version >= TEST_BRKVER(3, 8, 0, 0); + + rd_kafka_consumer_group_type_t group_protocol_in_use = + RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN; + rd_kafka_consumer_group_type_t group_protocol_not_in_use = + RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN; + + SUB_TEST_QUICK( + "%s ListConsumerGroups with %s, request_timeout %d" + ", match_states %s, match_types %s", + rd_kafka_name(rk), what, request_timeout, RD_STR_ToF(match_states), + RD_STR_ToF(match_types)); + + /* match_states would not work if broker version is below 2.7.0 */ + if (!has_match_states) + match_states = rd_false; + + /* match_types would not work if broker version is below 3.8.0 */ + if (!has_match_types) + match_types = rd_false; + + q = useq ? useq : rd_kafka_queue_new(rk); + + options = + rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS); + + if (request_timeout != -1) { + TEST_CALL_ERR__(rd_kafka_AdminOptions_set_request_timeout( + options, request_timeout, errstr, sizeof(errstr))); + } + + if (match_states) { + rd_kafka_consumer_group_state_t empty = + RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY; + + TEST_CALL_ERROR__( + rd_kafka_AdminOptions_set_match_consumer_group_states( + options, &empty, 1)); + } + + if (match_types) { + if (test_consumer_group_protocol_classic()) { + group_protocol_in_use = + RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC; + group_protocol_not_in_use = + RD_KAFKA_CONSUMER_GROUP_TYPE_CONSUMER; + } else { + group_protocol_in_use = + RD_KAFKA_CONSUMER_GROUP_TYPE_CONSUMER; + group_protocol_not_in_use = + RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC; + } + TEST_CALL_ERROR__( + rd_kafka_AdminOptions_set_match_consumer_group_types( + options, &group_protocol_in_use, 1)); + } + + topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1)); + exp_mdtopic.topic = topic; + + /* Create the topics first. */ + test_CreateTopics_simple(rk, NULL, &topic, 1, partitions_cnt, NULL); + + /* Verify that topics are reported by metadata */ + test_wait_metadata_update(rk, &exp_mdtopic, 1, NULL, 0, 15 * 1000); + + /* Produce 100 msgs */ + test_produce_msgs_easy(topic, testid, 0, msgs_cnt); + + for (i = 0; i < TEST_LIST_CONSUMER_GROUPS_CNT; i++) { + char *group = rd_strdup(test_mk_topic_name(__FUNCTION__, 1)); + test_consume_msgs_easy(group, topic, testid, -1, msgs_cnt, + NULL); + list_consumer_groups[i] = group; + } + + test_ListConsumerGroups_helper(rk, options, q, list_consumer_groups, + TEST_LIST_CONSUMER_GROUPS_CNT, + TEST_LIST_CONSUMER_GROUPS_CNT, + group_protocol_in_use, has_match_states); + + if (match_types) { + /* Simply test with the option of protocol not in use */ + rd_kafka_AdminOptions_t *option_group_protocol_not_in_use = + rd_kafka_AdminOptions_new( + rk, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS); + rd_kafka_AdminOptions_set_match_consumer_group_types( + option_group_protocol_not_in_use, + &group_protocol_not_in_use, 1); + + test_ListConsumerGroups_helper( + rk, option_group_protocol_not_in_use, q, + list_consumer_groups, TEST_LIST_CONSUMER_GROUPS_CNT, 0, + group_protocol_not_in_use, rd_false); + + rd_kafka_AdminOptions_destroy(option_group_protocol_not_in_use); + } test_DeleteGroups_simple(rk, NULL, (char **)list_consumer_groups, TEST_LIST_CONSUMER_GROUPS_CNT, NULL); @@ -3005,8 +3066,7 @@ static void do_test_ListConsumerGroups(const char *what, rd_free(topic); - if (options) - rd_kafka_AdminOptions_destroy(options); + rd_kafka_AdminOptions_destroy(options); if (!useq) rd_kafka_queue_destroy(q); @@ -5451,8 +5511,14 @@ static void do_test_apis(rd_kafka_type_t cltype) { do_test_DeleteRecords("main queue, op timeout 1500", rk, mainq, 1500); /* List groups */ - do_test_ListConsumerGroups("temp queue", rk, NULL, -1, rd_false); - do_test_ListConsumerGroups("main queue", rk, mainq, 1500, rd_true); + do_test_ListConsumerGroups("temp queue", rk, NULL, -1, rd_false, + rd_true); + do_test_ListConsumerGroups("temp queue", rk, NULL, -1, rd_false, + rd_false); + do_test_ListConsumerGroups("main queue", rk, mainq, 1500, rd_true, + rd_true); + do_test_ListConsumerGroups("main queue", rk, mainq, 1500, rd_true, + rd_false); /* TODO: check this test after KIP-848 admin operation * implementation */ diff --git a/lib/librdkafka-2.10.1/tests/0082-fetch_max_bytes.cpp b/lib/librdkafka-2.15.0/tests/0082-fetch_max_bytes.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0082-fetch_max_bytes.cpp rename to lib/librdkafka-2.15.0/tests/0082-fetch_max_bytes.cpp diff --git a/lib/librdkafka-2.10.1/tests/0083-cb_event.c b/lib/librdkafka-2.15.0/tests/0083-cb_event.c similarity index 99% rename from lib/librdkafka-2.10.1/tests/0083-cb_event.c rename to lib/librdkafka-2.15.0/tests/0083-cb_event.c index da8f3fd3a32..628fe0108e3 100644 --- a/lib/librdkafka-2.10.1/tests/0083-cb_event.c +++ b/lib/librdkafka-2.15.0/tests/0083-cb_event.c @@ -99,10 +99,10 @@ int main_0083_cb_event(int argc, char **argv) { rk_p = test_create_producer(); rkt_p = test_create_producer_topic(rk_p, topic, NULL); - test_wait_topic_exists(rk_p, topic, 5000); - err = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000)); + err = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000)); TEST_ASSERT(!err, "Topic auto creation failed: %s", rd_kafka_err2str(err)); + test_wait_topic_exists(rk_p, topic, 5000); test_conf_init(&conf, &tconf, 0); rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE); diff --git a/lib/librdkafka-2.10.1/tests/0084-destroy_flags.c b/lib/librdkafka-2.15.0/tests/0084-destroy_flags.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0084-destroy_flags.c rename to lib/librdkafka-2.15.0/tests/0084-destroy_flags.c diff --git a/lib/librdkafka-2.10.1/tests/0085-headers.cpp b/lib/librdkafka-2.15.0/tests/0085-headers.cpp similarity index 98% rename from lib/librdkafka-2.10.1/tests/0085-headers.cpp rename to lib/librdkafka-2.15.0/tests/0085-headers.cpp index 637f29916f5..785acb5310f 100644 --- a/lib/librdkafka-2.10.1/tests/0085-headers.cpp +++ b/lib/librdkafka-2.15.0/tests/0085-headers.cpp @@ -34,7 +34,6 @@ static RdKafka::Producer *producer; static RdKafka::KafkaConsumer *consumer; static std::string topic; -static bool first_produce = true; static void assert_all_headers_match(RdKafka::Headers *actual, const RdKafka::Headers *expected) { @@ -88,10 +87,6 @@ static void test_headers(RdKafka::Headers *produce_headers, Test::Fail("produce() failed: " + RdKafka::err2str(err)); producer->flush(tmout_multip(10 * 1000)); - if (first_produce) { - test_wait_topic_exists(producer->c_ptr(), topic.c_str(), 5000); - first_produce = false; - } if (producer->outq_len() > 0) Test::Fail(tostr() << "Expected producer to be flushed, " @@ -363,6 +358,8 @@ int main_0085_headers(int argc, char **argv) { delete conf; + Test::create_topic_wait_exists(p, topic.c_str(), 1, -1, 5000); + std::vector parts; parts.push_back(RdKafka::TopicPartition::create( topic, 0, RdKafka::Topic::OFFSET_BEGINNING)); @@ -385,6 +382,7 @@ int main_0085_headers(int argc, char **argv) { test_failed_produce(); test_assignment_op(); + Test::delete_topic(p, topic.c_str()); c->close(); delete c; delete p; diff --git a/lib/librdkafka-2.10.1/tests/0086-purge.c b/lib/librdkafka-2.15.0/tests/0086-purge.c similarity index 98% rename from lib/librdkafka-2.10.1/tests/0086-purge.c rename to lib/librdkafka-2.15.0/tests/0086-purge.c index 27605784335..115713fb8cd 100644 --- a/lib/librdkafka-2.10.1/tests/0086-purge.c +++ b/lib/librdkafka-2.15.0/tests/0086-purge.c @@ -80,14 +80,10 @@ static rd_kafka_resp_err_t on_request_sent(rd_kafka_t *rk, size_t size, void *ic_opaque) { - /* Save socket fd to limit ProduceRequest */ - if (ApiKey == RD_KAFKAP_ApiVersion) { - test_sockfd = sockfd; - return RD_KAFKA_RESP_ERR_NO_ERROR; - } if (ApiKey == RD_KAFKAP_Produce) { mtx_lock(&produce_sent_req_lock); produce_sent_req_cnt++; + test_sockfd = sockfd; if (produce_sent_req_cnt > 1) cnd_broadcast(&produce_sent_req_cnd); mtx_unlock(&produce_sent_req_lock); @@ -260,6 +256,10 @@ do_test_purge(const char *what, int remote, int idempotence, int gapless) { } rk = test_create_handle(RD_KAFKA_PRODUCER, conf); + if (remote) + /* Pre-create the topic to avoid unknown topic errors + * because of metadata propagation */ + test_create_topic_wait_exists(NULL, topic, 3, -1, 5000); TEST_SAY("Producing %d messages to topic %s\n", msgcnt, topic); diff --git a/lib/librdkafka-2.10.1/tests/0088-produce_metadata_timeout.c b/lib/librdkafka-2.15.0/tests/0088-produce_metadata_timeout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0088-produce_metadata_timeout.c rename to lib/librdkafka-2.15.0/tests/0088-produce_metadata_timeout.c diff --git a/lib/librdkafka-2.10.1/tests/0089-max_poll_interval.c b/lib/librdkafka-2.15.0/tests/0089-max_poll_interval.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0089-max_poll_interval.c rename to lib/librdkafka-2.15.0/tests/0089-max_poll_interval.c diff --git a/lib/librdkafka-2.10.1/tests/0090-idempotence.c b/lib/librdkafka-2.15.0/tests/0090-idempotence.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0090-idempotence.c rename to lib/librdkafka-2.15.0/tests/0090-idempotence.c diff --git a/lib/librdkafka-2.10.1/tests/0091-max_poll_interval_timeout.c b/lib/librdkafka-2.15.0/tests/0091-max_poll_interval_timeout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0091-max_poll_interval_timeout.c rename to lib/librdkafka-2.15.0/tests/0091-max_poll_interval_timeout.c diff --git a/lib/librdkafka-2.10.1/tests/0092-mixed_msgver.c b/lib/librdkafka-2.15.0/tests/0092-mixed_msgver.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0092-mixed_msgver.c rename to lib/librdkafka-2.15.0/tests/0092-mixed_msgver.c diff --git a/lib/librdkafka-2.10.1/tests/0093-holb.c b/lib/librdkafka-2.15.0/tests/0093-holb.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0093-holb.c rename to lib/librdkafka-2.15.0/tests/0093-holb.c diff --git a/lib/librdkafka-2.10.1/tests/0094-idempotence_msg_timeout.c b/lib/librdkafka-2.15.0/tests/0094-idempotence_msg_timeout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0094-idempotence_msg_timeout.c rename to lib/librdkafka-2.15.0/tests/0094-idempotence_msg_timeout.c diff --git a/lib/librdkafka-2.10.1/tests/0095-all_brokers_down.cpp b/lib/librdkafka-2.15.0/tests/0095-all_brokers_down.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0095-all_brokers_down.cpp rename to lib/librdkafka-2.15.0/tests/0095-all_brokers_down.cpp diff --git a/lib/librdkafka-2.10.1/tests/0097-ssl_verify.cpp b/lib/librdkafka-2.15.0/tests/0097-ssl_verify.cpp similarity index 94% rename from lib/librdkafka-2.10.1/tests/0097-ssl_verify.cpp rename to lib/librdkafka-2.15.0/tests/0097-ssl_verify.cpp index 1f944238861..64691dec6cf 100644 --- a/lib/librdkafka-2.10.1/tests/0097-ssl_verify.cpp +++ b/lib/librdkafka-2.15.0/tests/0097-ssl_verify.cpp @@ -126,8 +126,14 @@ class TestVerifyCb : public RdKafka::SslCertificateVerifyCb { class TestEventCb : public RdKafka::EventCb { public: bool should_succeed; - - TestEventCb(bool should_succeed) : should_succeed(should_succeed) { + /* True only when the broker is expected to reject the client's cert + * (untrusted client key + mutual TLS). Used to accept the broker's TLS + * alert only in that case. */ + bool expect_client_auth_fail; + + TestEventCb(bool should_succeed, bool expect_client_auth_fail) : + should_succeed(should_succeed), + expect_client_auth_fail(expect_client_auth_fail) { } void event_cb(RdKafka::Event &event) { @@ -147,14 +153,19 @@ class TestEventCb : public RdKafka::EventCb { else if (event.err() == RdKafka::ERR__SSL) { bool expected = false; Test::Say("SSL error: " + event.str() + "\n"); - if (event.str().find("alert number 42") != std::string::npos) - /* Verify that certificate isn't sent if not trusted - * by the broker. We should receive 42 (bad_certificate) - * instead of 46 (certificate_unknown). */ - expected = true; - else if (event.str().find("broker certificate could not be verified") != - std::string::npos) + if (expect_client_auth_fail) { + /* Broker rejects the untrusted client cert. Different broker TLS + * stacks send 42 (bad_certificate) or 40 (handshake_failure since + * Java 23). */ + if (event.str().find("alert number 42") != std::string::npos || + event.str().find("alert number 40") != std::string::npos) + expected = true; + } else if (event.str().find( + "broker certificate could not be verified") != + std::string::npos) { + /* Client couldn't verify the broker's cert. */ expected = true; + } if (!expected) Test::Fail("Unexpected SSL error message, got: " + event.str()); @@ -361,7 +372,12 @@ static void do_test_verify(const int line, bool should_succeed = verify_ok && (!untrusted_client_key || !is_client_auth_required() || security_protocol != "ssl"); - TestEventCb eventCb(should_succeed); + /* Broker rejects the client cert only when it's untrusted, client auth + * is required, and the endpoint is plain SSL. */ + bool expect_client_auth_fail = verify_ok && untrusted_client_key && + is_client_auth_required() && + security_protocol == "ssl"; + TestEventCb eventCb(should_succeed, expect_client_auth_fail); if (conf->set("event_cb", &eventCb, errstr) != RdKafka::Conf::CONF_OK) Test::Fail("Failed to set event_cb: " + errstr); diff --git a/lib/librdkafka-2.10.1/tests/0098-consumer-txn.cpp b/lib/librdkafka-2.15.0/tests/0098-consumer-txn.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0098-consumer-txn.cpp rename to lib/librdkafka-2.15.0/tests/0098-consumer-txn.cpp diff --git a/lib/librdkafka-2.10.1/tests/0099-commit_metadata.c b/lib/librdkafka-2.15.0/tests/0099-commit_metadata.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0099-commit_metadata.c rename to lib/librdkafka-2.15.0/tests/0099-commit_metadata.c diff --git a/lib/librdkafka-2.10.1/tests/0100-thread_interceptors.cpp b/lib/librdkafka-2.15.0/tests/0100-thread_interceptors.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0100-thread_interceptors.cpp rename to lib/librdkafka-2.15.0/tests/0100-thread_interceptors.cpp diff --git a/lib/librdkafka-2.10.1/tests/0101-fetch-from-follower.cpp b/lib/librdkafka-2.15.0/tests/0101-fetch-from-follower.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0101-fetch-from-follower.cpp rename to lib/librdkafka-2.15.0/tests/0101-fetch-from-follower.cpp diff --git a/lib/librdkafka-2.10.1/tests/0102-static_group_rebalance.c b/lib/librdkafka-2.15.0/tests/0102-static_group_rebalance.c similarity index 95% rename from lib/librdkafka-2.10.1/tests/0102-static_group_rebalance.c rename to lib/librdkafka-2.15.0/tests/0102-static_group_rebalance.c index 049ce16ca90..8f6c2a90c94 100644 --- a/lib/librdkafka-2.10.1/tests/0102-static_group_rebalance.c +++ b/lib/librdkafka-2.15.0/tests/0102-static_group_rebalance.c @@ -184,7 +184,6 @@ static void do_test_static_group_rebalance(void) { test_conf_set(conf, "group.instance.id", "consumer2"); c[1].rk = test_create_consumer(topic, rebalance_cb, rd_kafka_conf_dup(conf), NULL); - rd_kafka_conf_destroy(conf); test_wait_topic_exists(c[1].rk, topic, 5000); @@ -227,15 +226,15 @@ static void do_test_static_group_rebalance(void) { test_msgver_verify("first.verify", &mv, TEST_MSGVER_ALL, 0, msgcnt); TEST_SAY("== Testing consumer restart ==\n"); - conf = rd_kafka_conf_dup(rd_kafka_conf(c[1].rk)); /* Only c[1] should exhibit rebalance behavior */ c[1].expected_rb_event = RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS; TIMING_START(&t_close, "consumer restart"); test_consumer_close(c[1].rk); rd_kafka_destroy(c[1].rk); - - c[1].rk = test_create_handle(RD_KAFKA_CONSUMER, conf); + c[1].rk = test_create_consumer(topic, rebalance_cb, + rd_kafka_conf_dup(conf), NULL); + rd_kafka_conf_destroy(conf); rd_kafka_poll_set_consumer(c[1].rk); test_consumer_subscribe(c[1].rk, topics); @@ -673,16 +672,30 @@ static rd_bool_t is_api_key(rd_kafka_mock_request_t *request, void *opaque) { return rd_kafka_mock_request_api_key(request) == api_key; } +/** + * @enum do_test_static_membership_mock_variation_t + * @brief Variations of the static membership mock test. + */ +typedef enum do_test_static_membership_mock_variation_t { + /** Consumer 1 leaves with unsubscribe and rejoins the group */ + DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION_SAME_INSTANCE = 0, + /** Consumer 1 leaves with unsubscribe and a new consumer with same + * group.instance.id joins the group */ + DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION_NEW_INSTANCE = 1, + DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION__CNT +} do_test_static_membership_mock_variation_t; + /** * @brief Static group membership tests with the mock cluster. * Checks that consumer returns the same assignment * and generation id after re-joining. * - * variation 1) consumer 1 leaves and rejoins the group. - * variation 2) consumer 1 leaves and a new consumer with same - * group.instance.id joins the group. + * @param variation Test variation to run. + * + * @sa `do_test_static_membership_mock_variation_t` */ -static void do_test_static_membership_mock(int variation) { +static void do_test_static_membership_mock( + do_test_static_membership_mock_variation_t variation) { const char *bootstraps; rd_kafka_mock_cluster_t *mcluster; int32_t api_key = RD_KAFKAP_ConsumerGroupHeartbeat; @@ -693,8 +706,11 @@ static void do_test_static_membership_mock(int variation) { rd_kafka_topic_partition_list_t *prev_assignment1, *prev_assignment2, *next_assignment1, *next_assignment2; - SUB_TEST_QUICK("%s", variation == 1 ? "with unsubscribe" - : "with new instance"); + SUB_TEST_QUICK( + "%s", + variation == DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION_SAME_INSTANCE + ? "with same instance" + : "with new instance"); mcluster = test_mock_cluster_new(3, &bootstraps); rd_kafka_mock_topic_create(mcluster, topic, 2, 3); @@ -730,7 +746,8 @@ static void do_test_static_membership_mock(int variation) { &api_key); rd_kafka_mock_stop_request_tracking(mcluster); - if (variation == 2) { + if (variation == + DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION_NEW_INSTANCE) { /* Don't destroy it immediately because the * topic partition lists still hold a reference. */ consumer_1_to_destroy = consumer1; @@ -800,11 +817,7 @@ int main_0102_static_group_rebalance(int argc, char **argv) { int main_0102_static_group_rebalance_mock(int argc, char **argv) { TEST_SKIP_MOCK_CLUSTER(0); - - TEST_SKIP( - "Static membership mock fixes are present in another PR. Test it " - "with that PR.\n"); - return 0; + int variation; if (test_consumer_group_protocol_classic()) { TEST_SKIP( @@ -813,7 +826,11 @@ int main_0102_static_group_rebalance_mock(int argc, char **argv) { return 0; } - do_test_static_membership_mock(1); - do_test_static_membership_mock(2); + for (variation = DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION_SAME_INSTANCE; + variation < DO_TEST_STATIC_MEMBERSHIP_MOCK_VARIATION__CNT; + variation++) { + do_test_static_membership_mock(variation); + } + return 0; } diff --git a/lib/librdkafka-2.10.1/tests/0103-transactions.c b/lib/librdkafka-2.15.0/tests/0103-transactions.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0103-transactions.c rename to lib/librdkafka-2.15.0/tests/0103-transactions.c diff --git a/lib/librdkafka-2.10.1/tests/0104-fetch_from_follower_mock.c b/lib/librdkafka-2.15.0/tests/0104-fetch_from_follower_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0104-fetch_from_follower_mock.c rename to lib/librdkafka-2.15.0/tests/0104-fetch_from_follower_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0105-transactions_mock.c b/lib/librdkafka-2.15.0/tests/0105-transactions_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0105-transactions_mock.c rename to lib/librdkafka-2.15.0/tests/0105-transactions_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0106-cgrp_sess_timeout.c b/lib/librdkafka-2.15.0/tests/0106-cgrp_sess_timeout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0106-cgrp_sess_timeout.c rename to lib/librdkafka-2.15.0/tests/0106-cgrp_sess_timeout.c diff --git a/lib/librdkafka-2.10.1/tests/0107-topic_recreate.c b/lib/librdkafka-2.15.0/tests/0107-topic_recreate.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0107-topic_recreate.c rename to lib/librdkafka-2.15.0/tests/0107-topic_recreate.c diff --git a/lib/librdkafka-2.10.1/tests/0109-auto_create_topics.cpp b/lib/librdkafka-2.15.0/tests/0109-auto_create_topics.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0109-auto_create_topics.cpp rename to lib/librdkafka-2.15.0/tests/0109-auto_create_topics.cpp diff --git a/lib/librdkafka-2.10.1/tests/0110-batch_size.cpp b/lib/librdkafka-2.15.0/tests/0110-batch_size.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0110-batch_size.cpp rename to lib/librdkafka-2.15.0/tests/0110-batch_size.cpp diff --git a/lib/librdkafka-2.10.1/tests/0111-delay_create_topics.cpp b/lib/librdkafka-2.15.0/tests/0111-delay_create_topics.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0111-delay_create_topics.cpp rename to lib/librdkafka-2.15.0/tests/0111-delay_create_topics.cpp diff --git a/lib/librdkafka-2.10.1/tests/0112-assign_unknown_part.c b/lib/librdkafka-2.15.0/tests/0112-assign_unknown_part.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0112-assign_unknown_part.c rename to lib/librdkafka-2.15.0/tests/0112-assign_unknown_part.c diff --git a/lib/librdkafka-2.10.1/tests/0113-cooperative_rebalance.cpp b/lib/librdkafka-2.15.0/tests/0113-cooperative_rebalance.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0113-cooperative_rebalance.cpp rename to lib/librdkafka-2.15.0/tests/0113-cooperative_rebalance.cpp diff --git a/lib/librdkafka-2.10.1/tests/0114-sticky_partitioning.cpp b/lib/librdkafka-2.15.0/tests/0114-sticky_partitioning.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0114-sticky_partitioning.cpp rename to lib/librdkafka-2.15.0/tests/0114-sticky_partitioning.cpp diff --git a/lib/librdkafka-2.10.1/tests/0115-producer_auth.cpp b/lib/librdkafka-2.15.0/tests/0115-producer_auth.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0115-producer_auth.cpp rename to lib/librdkafka-2.15.0/tests/0115-producer_auth.cpp diff --git a/lib/librdkafka-2.10.1/tests/0116-kafkaconsumer_close.cpp b/lib/librdkafka-2.15.0/tests/0116-kafkaconsumer_close.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0116-kafkaconsumer_close.cpp rename to lib/librdkafka-2.15.0/tests/0116-kafkaconsumer_close.cpp diff --git a/lib/librdkafka-2.10.1/tests/0117-mock_errors.c b/lib/librdkafka-2.15.0/tests/0117-mock_errors.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0117-mock_errors.c rename to lib/librdkafka-2.15.0/tests/0117-mock_errors.c diff --git a/lib/librdkafka-2.10.1/tests/0118-commit_rebalance.c b/lib/librdkafka-2.15.0/tests/0118-commit_rebalance.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0118-commit_rebalance.c rename to lib/librdkafka-2.15.0/tests/0118-commit_rebalance.c diff --git a/lib/librdkafka-2.10.1/tests/0119-consumer_auth.cpp b/lib/librdkafka-2.15.0/tests/0119-consumer_auth.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0119-consumer_auth.cpp rename to lib/librdkafka-2.15.0/tests/0119-consumer_auth.cpp diff --git a/lib/librdkafka-2.10.1/tests/0120-asymmetric_subscription.c b/lib/librdkafka-2.15.0/tests/0120-asymmetric_subscription.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0120-asymmetric_subscription.c rename to lib/librdkafka-2.15.0/tests/0120-asymmetric_subscription.c diff --git a/lib/librdkafka-2.10.1/tests/0121-clusterid.c b/lib/librdkafka-2.15.0/tests/0121-clusterid.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0121-clusterid.c rename to lib/librdkafka-2.15.0/tests/0121-clusterid.c diff --git a/lib/librdkafka-2.10.1/tests/0122-buffer_cleaning_after_rebalance.c b/lib/librdkafka-2.15.0/tests/0122-buffer_cleaning_after_rebalance.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0122-buffer_cleaning_after_rebalance.c rename to lib/librdkafka-2.15.0/tests/0122-buffer_cleaning_after_rebalance.c diff --git a/lib/librdkafka-2.10.1/tests/0123-connections_max_idle.c b/lib/librdkafka-2.15.0/tests/0123-connections_max_idle.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0123-connections_max_idle.c rename to lib/librdkafka-2.15.0/tests/0123-connections_max_idle.c diff --git a/lib/librdkafka-2.10.1/tests/0124-openssl_invalid_engine.c b/lib/librdkafka-2.15.0/tests/0124-openssl_invalid_engine.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0124-openssl_invalid_engine.c rename to lib/librdkafka-2.15.0/tests/0124-openssl_invalid_engine.c diff --git a/lib/librdkafka-2.10.1/tests/0125-immediate_flush.c b/lib/librdkafka-2.15.0/tests/0125-immediate_flush.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0125-immediate_flush.c rename to lib/librdkafka-2.15.0/tests/0125-immediate_flush.c diff --git a/lib/librdkafka-2.15.0/tests/0126-oauthbearer_oidc.c b/lib/librdkafka-2.15.0/tests/0126-oauthbearer_oidc.c new file mode 100644 index 00000000000..0e040300357 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0126-oauthbearer_oidc.c @@ -0,0 +1,1164 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2021-2022, Magnus Edenhill + * 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +/* Typical include path would be , but this program + * is built from within the librdkafka source tree and thus differs. */ +#include "rdkafka.h" /* for Kafka driver */ + +#define TEST_FIXTURES_FOLDER "./fixtures" +#define TEST_FIXTURES_OAUTHBEARER_FOLDER TEST_FIXTURES_FOLDER "/oauthbearer/" +#define TEST_FIXTURES_JWT_ASSERTION_TEMPLATE \ + TEST_FIXTURES_OAUTHBEARER_FOLDER "jwt_assertion_template.json" + +static rd_bool_t error_seen; +static rd_bool_t user_token_cb_called; +/** + * @brief After config OIDC, make sure the producer and consumer + * can work successfully. + * + */ +static void +do_test_produce_consumer_with_OIDC(const char *test_name, + const rd_kafka_conf_t *base_conf) { + const char *topic; + uint64_t testid; + rd_kafka_t *p1; + rd_kafka_t *c1; + rd_kafka_conf_t *conf; + + const char *url = test_getenv("VALID_OIDC_URL", NULL); + + SUB_TEST("Test producer and consumer with oidc configuration: %s", + test_name); + + if (!url) { + SUB_TEST_SKIP( + "VALID_OIDC_URL environment variable is not set\n"); + return; + } + + conf = rd_kafka_conf_dup(base_conf); + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", url); + + testid = test_id_generate(); + + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + p1 = test_create_handle(RD_KAFKA_PRODUCER, rd_kafka_conf_dup(conf)); + + topic = test_mk_topic_name("0126-oauthbearer_oidc", 1); + test_create_topic_wait_exists(p1, topic, 1, 3, 5000); + TEST_SAY("Topic: %s is created\n", topic); + + test_produce_msgs2(p1, topic, testid, 0, 0, 1, NULL, 0); + + test_conf_set(conf, "auto.offset.reset", "earliest"); + c1 = test_create_consumer(topic, NULL, conf, NULL); + test_consumer_subscribe(c1, topic); + + /* Give it some time to trigger the token refresh. */ + rd_usleep(5 * 1000 * 1000, NULL); + test_consumer_poll("OIDC.C1", c1, testid, 1, -1, 1, NULL); + + test_consumer_close(c1); + + rd_kafka_destroy(p1); + rd_kafka_destroy(c1); + SUB_TEST_PASS(); +} + + +/** + * @brief After config OIDC, make sure the share consumer + * can work successfully. + */ +static void +do_test_produce_share_consumer_with_OIDC(const char *test_name, + const rd_kafka_conf_t *base_conf) { + const char *topic; + rd_kafka_t *p1; + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + const char *grp_conf[] = {"share.auto.offset.reset", "SET", "earliest"}; + int consumed = 0, attempts; + const int msg_cnt = 10; + + const char *url = test_getenv("VALID_OIDC_URL", NULL); + + SUB_TEST("Test share consumer with oidc configuration: %s", test_name); + + if (!url) { + SUB_TEST_SKIP( + "VALID_OIDC_URL environment variable is not set\n"); + return; + } + + conf = rd_kafka_conf_dup(base_conf); + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", url); + + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + p1 = test_create_handle(RD_KAFKA_PRODUCER, conf); + + topic = test_mk_topic_name("0126-oauthbearer_oidc_share", 1); + test_create_topic_wait_exists(p1, topic, 1, 3, 5000); + TEST_SAY("Topic: %s is created\n", topic); + + test_produce_msgs_easy(topic, 0, 0, msg_cnt); + + /* Create share consumer (picks up SASL config from test_conf_init) */ + sc1 = test_create_share_consumer("oidc-share-group", NULL); + + /* Set group config for earliest offset */ + test_IncrementalAlterConfigs_simple(p1, RD_KAFKA_RESOURCE_GROUP, + "oidc-share-group", grp_conf, 1); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Give it some time to trigger the token refresh. */ + rd_usleep(5 * 1000 * 1000, NULL); + + /* Consume messages */ + attempts = 50; + while (consumed < msg_cnt && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + + err = rd_kafka_share_poll(sc1, 3000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + if (!rd_kafka_messages_get(batch, m)->err) + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == msg_cnt, "Expected %d messages, consumed %d", + msg_cnt, consumed); + TEST_SAY("Share consumer consumed %d/%d messages with OIDC\n", consumed, + msg_cnt); + + rd_kafka_share_consumer_close(sc1); + rd_kafka_share_destroy(sc1); + rd_kafka_destroy(p1); + SUB_TEST_PASS(); +} + + +static void +auth_error_cb(rd_kafka_t *rk, int err, const char *reason, void *opaque) { + if (err == RD_KAFKA_RESP_ERR__AUTHENTICATION || + err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) { + TEST_SAY("Expected error: %s: %s\n", rd_kafka_err2str(err), + reason); + error_seen = rd_true; + } else + TEST_FAIL("Unexpected error: %s: %s", rd_kafka_err2str(err), + reason); + rd_kafka_yield(rk); +} + +/** + * @brief Test-only token-refresh callback used to verify that a + * user-registered callback pre-empts the built-in OIDC callbacks. + */ +static void user_token_refresh_cb(rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque) { + user_token_cb_called = rd_true; + rd_kafka_oauthbearer_set_token_failure( + rk, "aws_iam test user token refresh cb"); +} + + +/** + * @brief After config OIDC, if the token is expired, make sure + * the authentication fail as expected. + * + */ +static void do_test_produce_consumer_with_OIDC_expired_token_should_fail( + const rd_kafka_conf_t *base_conf) { + rd_kafka_t *c1; + uint64_t testid; + rd_kafka_conf_t *conf; + + const char *expired_url = test_getenv("EXPIRED_TOKEN_OIDC_URL", NULL); + + SUB_TEST("Test OAUTHBEARER/OIDC failing with expired JWT"); + + if (!expired_url) { + SUB_TEST_SKIP( + "EXPIRED_TOKEN_OIDC_URL environment variable is not set\n"); + return; + } + + conf = rd_kafka_conf_dup(base_conf); + + error_seen = rd_false; + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", expired_url); + + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + testid = test_id_generate(); + + c1 = test_create_consumer("OIDC.fail.C1", NULL, conf, NULL); + + test_consumer_poll_no_msgs("OIDC.fail.C1", c1, testid, 10 * 1000); + TEST_ASSERT(error_seen); + + test_consumer_close(c1); + rd_kafka_destroy(c1); + SUB_TEST_PASS(); +} + +/** + * @brief After config OIDC with expired token, make sure the share consumer + * authentication fails as expected. + */ +static void do_test_produce_share_consumer_with_OIDC_expired_token_should_fail( + const rd_kafka_conf_t *base_conf) { + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + rd_kafka_error_t *err; + rd_kafka_topic_partition_list_t *subs; + int attempts; + char errstr[512]; + + const char *expired_url = test_getenv("EXPIRED_TOKEN_OIDC_URL", NULL); + + SUB_TEST( + "Test OAUTHBEARER/OIDC share consumer failing with " + "expired JWT"); + + if (!expired_url) { + SUB_TEST_SKIP( + "EXPIRED_TOKEN_OIDC_URL environment variable is not " + "set\n"); + return; + } + + conf = rd_kafka_conf_dup(base_conf); + + error_seen = rd_false; + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", expired_url); + test_conf_set(conf, "group.id", "oidc-share-fail"); + + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + sc1 = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(sc1, "Failed to create share consumer: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, "oidc-share-fail-topic", + RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 10; + while (!error_seen && attempts-- > 0) { + err = rd_kafka_share_poll(sc1, 3000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "Expected no messages with expired token, got %zu", + rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(error_seen, + "Expected authentication error for share consumer " + "with expired token"); + + rd_kafka_share_consumer_close(sc1); + rd_kafka_share_destroy(sc1); + SUB_TEST_PASS(); +} + + +/** + * @brief After configiguring OIDC, make sure the + * authentication fails as expected. + */ +static void do_test_produce_consumer_with_OIDC_should_fail( + const char *test_name, + const rd_kafka_conf_t *base_conf) { + rd_kafka_t *c1; + uint64_t testid; + rd_kafka_conf_t *conf; + + const char *url = test_getenv("VALID_OIDC_URL", NULL); + + SUB_TEST("Test authentication failure with oidc configuration: %s", + test_name); + if (!url) { + SUB_TEST_SKIP( + "VALID_OIDC_URL environment variable is not set\n"); + return; + } + + error_seen = rd_false; + + conf = rd_kafka_conf_dup(base_conf); + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", url); + + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + testid = test_id_generate(); + + c1 = test_create_consumer("OIDC.fail.C1", NULL, conf, NULL); + + test_consumer_poll_no_msgs("OIDC.fail.C1", c1, testid, 5 * 1000); + + TEST_ASSERT(error_seen); + + test_consumer_close(c1); + rd_kafka_destroy(c1); + SUB_TEST_PASS(); +} + +/** + * @brief After config OIDC, if the token endpoint is not valid, make sure the + * authentication fail as expected. + * + */ +static void +do_test_produce_consumer_with_OIDC_should_fail_invalid_token_endpoint( + const rd_kafka_conf_t *base_conf) { + rd_kafka_t *c1; + uint64_t testid; + rd_kafka_conf_t *conf; + + const char *invalid_url = test_getenv("INVALID_OIDC_URL", NULL); + + SUB_TEST("Test OAUTHBEARER/OIDC failing with invalid JWT"); + + if (!invalid_url) { + SUB_TEST_SKIP( + "INVALID_OIDC_URL environment variable is not set\n"); + return; + } + + conf = rd_kafka_conf_dup(base_conf); + + error_seen = rd_false; + + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", invalid_url); + + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + testid = test_id_generate(); + + c1 = test_create_consumer("OIDC.fail.C1", NULL, conf, NULL); + + test_consumer_poll_no_msgs("OIDC.fail.C1", c1, testid, 10 * 1000); + + TEST_ASSERT(error_seen); + + test_consumer_close(c1); + rd_kafka_destroy(c1); + SUB_TEST_PASS(); +} + + +/** + * @brief After config OIDC with invalid token endpoint, make sure the + * share consumer authentication fails as expected. + */ +static void +do_test_produce_share_consumer_with_OIDC_should_fail_invalid_token_endpoint( + const rd_kafka_conf_t *base_conf) { + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + rd_kafka_error_t *err; + rd_kafka_topic_partition_list_t *subs; + int attempts; + char errstr[512]; + + const char *invalid_url = test_getenv("INVALID_OIDC_URL", NULL); + + SUB_TEST( + "Test OAUTHBEARER/OIDC share consumer failing with " + "invalid JWT"); + + if (!invalid_url) { + SUB_TEST_SKIP( + "INVALID_OIDC_URL environment variable is not set\n"); + return; + } + + conf = rd_kafka_conf_dup(base_conf); + + error_seen = rd_false; + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", invalid_url); + test_conf_set(conf, "group.id", "oidc-share-fail-invalid"); + + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + sc1 = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(sc1, "Failed to create share consumer: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, "oidc-share-fail-invalid-topic", + RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 10; + while (!error_seen && attempts-- > 0) { + err = rd_kafka_share_poll(sc1, 3000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "Expected no messages with invalid token " + "endpoint, got %zu", + rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(error_seen, + "Expected authentication error for share consumer " + "with invalid token endpoint"); + + rd_kafka_share_consumer_close(sc1); + rd_kafka_share_destroy(sc1); + SUB_TEST_PASS(); +} + +typedef enum oidc_configuration_jwt_bearer_variation_t { + /** Use a private key file. */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE, + /** Use an encrypted private key file. */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE_ENCRYPTED, + /** Use a private key file + * set as a configuration property. */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_STRING, + /** Use an encrypted private key file + * set as a configuration property. */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_STRING_ENCRYPTED, + /** Use a private key file + * and a template for the JWT assertion. */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_TEMPLATE_FILE, + /** Use a private key file and set the JOSE algorithm to ES256. + * This variation will fail as the private key is RSA in trivup */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_JOSE_ALGORITHM_ES256, + /** Invalid scope */ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_INVALID_SCOPE, + OIDC_CONFIGURATION_JWT_BEARER_VARIATION__CNT +} oidc_configuration_jwt_bearer_variation_t; + +#define OIDC_CONFIGURATION_JWT_BEARER_VARIATION__FIRST_FAILING \ + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_JOSE_ALGORITHM_ES256 + +static const char *oidc_configuration_jwt_bearer_variation_name( + oidc_configuration_jwt_bearer_variation_t variation) { + rd_assert( + variation >= + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE && + variation < OIDC_CONFIGURATION_JWT_BEARER_VARIATION__CNT); + static const char *names[] = { + "private key file", "private key encrypted file", + "private key pem string", "private key encrypted pem string", + "template file", "JOSE algorithm ES256", + "invalid scope"}; + return names[variation]; +} + +static rd_kafka_conf_t *oidc_configuration_jwt_bearer( + rd_kafka_conf_t *conf, + oidc_configuration_jwt_bearer_variation_t variation) { + char file_content[4096]; + const char *private_key_file = + test_getenv("OAUTHBEARER_CLIENT_PRIVATE_KEY", NULL); + const char *private_key_encrypted_file = + test_getenv("OAUTHBEARER_CLIENT_PRIVATE_KEY_ENCRYPTED", NULL); + const char *private_key_password = + test_getenv("OAUTHBEARER_CLIENT_PRIVATE_KEY_PASSWORD", NULL); + + conf = rd_kafka_conf_dup(conf); + test_conf_set(conf, "sasl.oauthbearer.grant.type", + "urn:ietf:params:oauth:grant-type:jwt-bearer"); + /* "sub" isn't mandatory if already defined in the template. */ + if (variation != OIDC_CONFIGURATION_JWT_BEARER_VARIATION_TEMPLATE_FILE) + test_conf_set(conf, "sasl.oauthbearer.assertion.claim.sub", + "testuser"); + else + test_conf_set(conf, + "sasl.oauthbearer.assertion.jwt.template.file", + TEST_FIXTURES_JWT_ASSERTION_TEMPLATE); + + if (variation == + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_JOSE_ALGORITHM_ES256) + test_conf_set(conf, "sasl.oauthbearer.assertion.algorithm", + "ES256"); + if (variation == OIDC_CONFIGURATION_JWT_BEARER_VARIATION_INVALID_SCOPE) + test_conf_set(conf, "sasl.oauthbearer.scope", "invalid_scope"); + + switch (variation) { + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE: + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_TEMPLATE_FILE: + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_JOSE_ALGORITHM_ES256: + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_INVALID_SCOPE: + if (!private_key_file) + goto fail; + + test_conf_set(conf, + "sasl.oauthbearer.assertion.private.key.file", + private_key_file); + break; + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE_ENCRYPTED: + if (!private_key_encrypted_file || !private_key_password) + goto fail; + test_conf_set(conf, + "sasl.oauthbearer.assertion.private.key.file", + private_key_encrypted_file); + test_conf_set( + conf, "sasl.oauthbearer.assertion.private.key.passphrase", + private_key_password); + break; + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_STRING: + if (!private_key_file) + goto fail; + TEST_ASSERT(test_read_file(private_key_file, file_content, + sizeof(file_content)) > 0); + + test_conf_set(conf, + "sasl.oauthbearer.assertion.private.key.pem", + file_content); + break; + case OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_STRING_ENCRYPTED: + if (!private_key_encrypted_file || !private_key_password) + goto fail; + TEST_ASSERT(test_read_file(private_key_file, file_content, + sizeof(file_content)) > 0); + + test_conf_set(conf, + "sasl.oauthbearer.assertion.private.key.pem", + file_content); + test_conf_set( + conf, "sasl.oauthbearer.assertion.private.key.passphrase", + private_key_password); + break; + default: + rd_assert(!*"Unknown OIDC JWT bearer test variation"); + } + return conf; +fail: + rd_kafka_conf_destroy(conf); + TEST_WARN("Skipping OIDC JWT bearer test variation: %s", + oidc_configuration_jwt_bearer_variation_name(variation)); + return NULL; +} + +/** + * @brief Share consumer version of + * do_test_produce_consumer_with_OIDC_should_fail. Verifies authentication + * failure with share consumer. + */ +static void do_test_produce_share_consumer_with_OIDC_should_fail( + const char *test_name, + const rd_kafka_conf_t *base_conf) { + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + rd_kafka_error_t *err; + rd_kafka_topic_partition_list_t *subs; + int attempts; + char errstr[512]; + + const char *url = test_getenv("VALID_OIDC_URL", NULL); + + SUB_TEST("Test share consumer auth failure with oidc configuration: %s", + test_name); + if (!url) { + SUB_TEST_SKIP( + "VALID_OIDC_URL environment variable is not set\n"); + return; + } + + error_seen = rd_false; + + conf = rd_kafka_conf_dup(base_conf); + test_conf_set(conf, "sasl.oauthbearer.token.endpoint.url", url); + test_conf_set(conf, "group.id", "oidc-share-should-fail"); + + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + sc1 = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(sc1, "Failed to create share consumer: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, "oidc-share-should-fail-topic", + RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 10; + while (!error_seen && attempts-- > 0) { + err = rd_kafka_share_poll(sc1, 3000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "Expected no messages on auth failure, got %zu", + rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(error_seen, + "Expected authentication error for share consumer"); + + rd_kafka_share_consumer_close(sc1); + rd_kafka_share_destroy(sc1); + SUB_TEST_PASS(); +} + + +void do_test_produce_consumer_with_OIDC_jwt_bearer(rd_kafka_conf_t *conf) { + rd_kafka_conf_t *jwt_bearer_conf; + oidc_configuration_jwt_bearer_variation_t variation; + for (variation = + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE; + variation < OIDC_CONFIGURATION_JWT_BEARER_VARIATION__CNT; + variation++) { + const char *test_name; + jwt_bearer_conf = + oidc_configuration_jwt_bearer(conf, variation); + if (!jwt_bearer_conf) + continue; + + test_name = tsprintf( + "JWT bearer: %s\n", + oidc_configuration_jwt_bearer_variation_name(variation)); + + if (variation < + OIDC_CONFIGURATION_JWT_BEARER_VARIATION__FIRST_FAILING) + do_test_produce_consumer_with_OIDC(test_name, + jwt_bearer_conf); + else + do_test_produce_consumer_with_OIDC_should_fail( + test_name, jwt_bearer_conf); + rd_kafka_conf_destroy(jwt_bearer_conf); + } +} + +void do_test_produce_share_consumer_with_OIDC_jwt_bearer( + rd_kafka_conf_t *conf) { + rd_kafka_conf_t *jwt_bearer_conf; + oidc_configuration_jwt_bearer_variation_t variation; + for (variation = + OIDC_CONFIGURATION_JWT_BEARER_VARIATION_PRIVATE_KEY_FILE; + variation < OIDC_CONFIGURATION_JWT_BEARER_VARIATION__CNT; + variation++) { + const char *test_name; + jwt_bearer_conf = + oidc_configuration_jwt_bearer(conf, variation); + if (!jwt_bearer_conf) + continue; + + test_name = tsprintf( + "JWT bearer: %s\n", + oidc_configuration_jwt_bearer_variation_name(variation)); + + if (variation < + OIDC_CONFIGURATION_JWT_BEARER_VARIATION__FIRST_FAILING) + do_test_produce_share_consumer_with_OIDC( + test_name, jwt_bearer_conf); + else + do_test_produce_share_consumer_with_OIDC_should_fail( + test_name, jwt_bearer_conf); + rd_kafka_conf_destroy(jwt_bearer_conf); + } +} + + +typedef enum oidc_configuration_metadata_authentication_variation_t { + /** Azure IMDS. Successful case. */ + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_SUCCESS, + /** Azure IMDS. Missing client ID. */ + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_CLIENT_ID, + /** Azure IMDS. Missing resource parameter. */ + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_RESOURCE, + /** Azure IMDS. Missing API version. */ + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_API_VERSION, + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__CNT +} oidc_configuration_metadata_authentication_variation_t; + +#define OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__FIRST_FAILING \ + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_CLIENT_ID + +static const char *oidc_configuration_metadata_authentication_variation_name( + oidc_configuration_metadata_authentication_variation_t variation) { + rd_assert( + variation >= + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_SUCCESS && + variation < + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__CNT); + static const char *names[] = { + "Azure IMDS: success", "Azure IMDS: missing client ID", + "Azure IMDS: missing resource", "Azure IMDS: missing API version"}; + return names[variation]; +} + +static rd_kafka_conf_t *oidc_configuration_metadata_authentication( + rd_kafka_conf_t *conf, + oidc_configuration_metadata_authentication_variation_t variation) { + conf = rd_kafka_conf_dup(conf); + switch (variation) { + case OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_SUCCESS: + test_conf_set(conf, + "sasl.oauthbearer.metadata.authentication.type", + "azure_imds"); + test_conf_set(conf, "sasl.oauthbearer.config", + "query=__metadata_authentication_type=azure_imds&" + "api-version=2025-04-07&resource=" + "api://external_resource_id&client_id=client_id"); + break; + case OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_CLIENT_ID: + test_conf_set(conf, + "sasl.oauthbearer.metadata.authentication.type", + "azure_imds"); + test_conf_set(conf, "sasl.oauthbearer.config", + "query=__metadata_authentication_type=azure_imds&" + "api-version=2025-04-07&resource=" + "api://external_resource_id"); + break; + case OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_RESOURCE: + test_conf_set(conf, + "sasl.oauthbearer.metadata.authentication.type", + "azure_imds"); + test_conf_set(conf, "sasl.oauthbearer.config", + "query=__metadata_authentication_type=azure_imds&" + "api-version=2025-04-07&" + "client_id=client_id"); + break; + case OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_MISSING_API_VERSION: + test_conf_set(conf, + "sasl.oauthbearer.metadata.authentication.type", + "azure_imds"); + test_conf_set(conf, "sasl.oauthbearer.config", + "query=__metadata_authentication_type=azure_imds&" + "resource=" + "api://external_resource_id&client_id=client_id"); + break; + default: + TEST_ASSERT(rd_false, + "Unknown OIDC metadata authentication type"); + } + return conf; +} + +/* Test metadata-based authentication cases against Trivup. */ +void do_test_produce_consumer_with_OIDC_metadata_authentication( + rd_kafka_conf_t *conf) { + rd_kafka_conf_t *metadata_authentication_conf; + oidc_configuration_metadata_authentication_variation_t variation; + for ( + variation = + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_SUCCESS; + variation < + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__CNT; + variation++) { + const char *test_name; + metadata_authentication_conf = + oidc_configuration_metadata_authentication(conf, variation); + + test_name = tsprintf( + "Metadata authentication variation: %s\n", + oidc_configuration_metadata_authentication_variation_name( + variation)); + + if (variation < + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__FIRST_FAILING) + do_test_produce_consumer_with_OIDC( + test_name, metadata_authentication_conf); + else + do_test_produce_consumer_with_OIDC_should_fail( + test_name, metadata_authentication_conf); + rd_kafka_conf_destroy(metadata_authentication_conf); + } +} + +void do_test_produce_share_consumer_with_OIDC_metadata_authentication( + rd_kafka_conf_t *conf) { + rd_kafka_conf_t *metadata_authentication_conf; + oidc_configuration_metadata_authentication_variation_t variation; + for ( + variation = + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION_AZURE_IMDS_SUCCESS; + variation < + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__CNT; + variation++) { + const char *test_name; + metadata_authentication_conf = + oidc_configuration_metadata_authentication(conf, variation); + + test_name = tsprintf( + "Metadata authentication variation: %s\n", + oidc_configuration_metadata_authentication_variation_name( + variation)); + + if (variation < + OIDC_CONFIGURATION_METADATA_AUTHENTICATION_VARIATION__FIRST_FAILING) + do_test_produce_share_consumer_with_OIDC( + test_name, metadata_authentication_conf); + else + do_test_produce_share_consumer_with_OIDC_should_fail( + test_name, metadata_authentication_conf); + rd_kafka_conf_destroy(metadata_authentication_conf); + } +} + +typedef enum oidc_configuration_sub_claim_variation_t { + /** Use default "sub" claim (backward compatibility). */ + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_DEFAULT_SUB, + /** Explicitly set "sub" as the claim name. */ + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_EXPLICIT_SUB, + /** Use custom claim name "client_id". */ + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_CUSTOM_CLIENT_ID, + /** Set empty string "" — resets to default "sub" per librdkafka string + semantics. */ + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_EMPTY_STRING, + /** Use a claim name that doesn't exist in the token (should fail). */ + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_MISSING_CLAIM, + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__CNT +} oidc_configuration_sub_claim_variation_t; + +#define OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__FIRST_FAILING \ + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_MISSING_CLAIM + +static const char *oidc_configuration_sub_claim_variation_name( + oidc_configuration_sub_claim_variation_t variation) { + rd_assert(variation >= + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_DEFAULT_SUB && + variation < OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__CNT); + static const char *names[] = { + "default sub claim", "explicit sub claim", "custom client_id claim", + "empty string (defaults to sub)", "missing claim (should fail)"}; + return names[variation]; +} + +/** + * @brief Configure OIDC with different subject claim name variations. + * + * Note: This test assumes the OIDC token provider returns tokens with + * standard claims including "sub" and "client_id". + * The test validates that librdkafka can extract the subject from + * different claims based on configuration. + */ +static rd_kafka_conf_t *oidc_configuration_sub_claim( + rd_kafka_conf_t *conf, + oidc_configuration_sub_claim_variation_t variation) { + conf = rd_kafka_conf_dup(conf); + + switch (variation) { + case OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_DEFAULT_SUB: + break; + case OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_EXPLICIT_SUB: + test_conf_set(conf, "sasl.oauthbearer.sub.claim.name", "sub"); + break; + case OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_CUSTOM_CLIENT_ID: + test_conf_set(conf, "sasl.oauthbearer.sub.claim.name", + "client_id"); + break; + case OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_EMPTY_STRING: + test_conf_set(conf, "sasl.oauthbearer.sub.claim.name", ""); + break; + case OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_MISSING_CLAIM: + test_conf_set(conf, "sasl.oauthbearer.sub.claim.name", + "nonexistent_claim"); + break; + default: + rd_assert(!*"Unknown OIDC sub claim test variation"); + } + return conf; +} + +/** + * @brief Test producer and consumer with different subject claim name + * configurations. + * + * This test validates KIP-768 parity for sasl.oauthbearer.sub.claim.name: + * - Default behavior uses "sub" claim + * - Custom claim names can be configured + * - Missing configured claim causes validation failure + * - Non-empty claim value is enforced + */ +void do_test_produce_consumer_with_OIDC_sub_claim(rd_kafka_conf_t *conf) { + rd_kafka_conf_t *sub_claim_conf; + oidc_configuration_sub_claim_variation_t variation; + + const char *url = test_getenv("VALID_OIDC_URL", NULL); + + /* Check if we should skip sub claim tests based on environment */ + if (!url) { + SUB_TEST_SKIP( + "VALID_OIDC_URL environment variable is not set, " + "skipping sub claim tests\n"); + return; + } + + for (variation = OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_DEFAULT_SUB; + variation < OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__CNT; + variation++) { + const char *test_name; + sub_claim_conf = oidc_configuration_sub_claim(conf, variation); + + test_name = tsprintf( + "Sub claim variation: %s\n", + oidc_configuration_sub_claim_variation_name(variation)); + + if (variation < + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__FIRST_FAILING) { + /* These variations should succeed */ + do_test_produce_consumer_with_OIDC(test_name, + sub_claim_conf); + } else { + /* These variations should fail */ + do_test_produce_consumer_with_OIDC_should_fail( + test_name, sub_claim_conf); + } + rd_kafka_conf_destroy(sub_claim_conf); + } +} + +void do_test_produce_share_consumer_with_OIDC_sub_claim(rd_kafka_conf_t *conf) { + rd_kafka_conf_t *sub_claim_conf; + oidc_configuration_sub_claim_variation_t variation; + + const char *url = test_getenv("VALID_OIDC_URL", NULL); + + if (!url) { + SUB_TEST_SKIP( + "VALID_OIDC_URL environment variable is not set, " + "skipping share consumer sub claim tests\n"); + return; + } + + for (variation = OIDC_CONFIGURATION_SUB_CLAIM_VARIATION_DEFAULT_SUB; + variation < OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__CNT; + variation++) { + const char *test_name; + sub_claim_conf = oidc_configuration_sub_claim(conf, variation); + + test_name = tsprintf( + "Sub claim variation: %s\n", + oidc_configuration_sub_claim_variation_name(variation)); + + if (variation < + OIDC_CONFIGURATION_SUB_CLAIM_VARIATION__FIRST_FAILING) { + do_test_produce_share_consumer_with_OIDC( + test_name, sub_claim_conf); + } else { + do_test_produce_share_consumer_with_OIDC_should_fail( + test_name, sub_claim_conf); + } + rd_kafka_conf_destroy(sub_claim_conf); + } +} + +/** + * @brief aws_iam round-trips through rd_kafka_conf_set/get, confirming + * the s2i wiring for the new enum value. + */ +static void do_test_OIDC_aws_iam_conf_roundtrip(void) { + rd_kafka_conf_t *conf; + char errstr[512]; + char value[64]; + size_t value_size = sizeof(value); + + SUB_TEST("aws_iam round-trips through rd_kafka_conf_set/get"); + + conf = rd_kafka_conf_new(); + + TEST_ASSERT(rd_kafka_conf_set( + conf, "sasl.oauthbearer.metadata.authentication.type", + "aws_iam", errstr, sizeof(errstr)) == RD_KAFKA_CONF_OK, + "Failed to set aws_iam: %s", errstr); + + TEST_ASSERT(rd_kafka_conf_get( + conf, "sasl.oauthbearer.metadata.authentication.type", + value, &value_size) == RD_KAFKA_CONF_OK, + "Failed to read aws_iam back from conf"); + TEST_ASSERT(strcmp(value, "aws_iam") == 0, + "Expected 'aws_iam', got '%s'", value); + + rd_kafka_conf_destroy(conf); + SUB_TEST_PASS(); +} + +/** + * @brief When method=oidc and metadata.authentication.type=aws_iam + * but no token-refresh callback is registered, the built-in + * stub installed by librdkafka surfaces a token-refresh + * failure rather than silently falling back to a different + * OIDC flow. + */ +static void do_test_OIDC_aws_iam_stub_fires_without_callback( + const rd_kafka_conf_t *base_conf) { + rd_kafka_t *c1; + uint64_t testid; + rd_kafka_conf_t *conf; + + SUB_TEST("aws_iam stub fails token refresh when no callback is set"); + + conf = rd_kafka_conf_dup(base_conf); + test_conf_set(conf, "sasl.oauthbearer.metadata.authentication.type", + "aws_iam"); + + error_seen = rd_false; + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + testid = test_id_generate(); + + c1 = test_create_consumer("aws_iam.stub.C1", NULL, conf, NULL); + + test_consumer_poll_no_msgs("aws_iam.stub.C1", c1, testid, 5 * 1000); + TEST_ASSERT(error_seen, + "Expected aws_iam stub to surface a token refresh " + "failure"); + + test_consumer_close(c1); + rd_kafka_destroy(c1); + SUB_TEST_PASS(); +} + +/** + * @brief User-registered token refresh callback takes precedence over + * the aws_iam stub. + */ +static void +do_test_OIDC_aws_iam_user_callback_wins(const rd_kafka_conf_t *base_conf) { + rd_kafka_t *c1; + uint64_t testid; + rd_kafka_conf_t *conf; + + SUB_TEST( + "aws_iam: user-registered token refresh callback wins over stub"); + + conf = rd_kafka_conf_dup(base_conf); + test_conf_set(conf, "sasl.oauthbearer.metadata.authentication.type", + "aws_iam"); + + rd_kafka_conf_set_oauthbearer_token_refresh_cb(conf, + user_token_refresh_cb); + + user_token_cb_called = rd_false; + error_seen = rd_false; + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + + testid = test_id_generate(); + + c1 = test_create_consumer("aws_iam.user_cb.C1", NULL, conf, NULL); + + test_consumer_poll_no_msgs("aws_iam.user_cb.C1", c1, testid, 5 * 1000); + TEST_ASSERT(user_token_cb_called, + "Expected user-registered token refresh callback to " + "fire instead of the aws_iam stub"); + TEST_ASSERT(error_seen, + "Expected user-callback's set_token_failure to surface " + "as an authentication error"); + + test_consumer_close(c1); + rd_kafka_destroy(c1); + SUB_TEST_PASS(); +} + +int main_0126_oauthbearer_oidc(int argc, char **argv) { + rd_kafka_conf_t *conf; + const char *sec; + const char *oidc; + + test_conf_init(&conf, NULL, 60); + + sec = test_conf_get(conf, "security.protocol"); + if (!strstr(sec, "sasl")) { + TEST_SKIP("Apache Kafka cluster not configured for SASL\n"); + rd_kafka_conf_destroy(conf); + return 0; + } + + oidc = test_conf_get(conf, "sasl.oauthbearer.method"); + if (rd_strcasecmp(oidc, "OIDC")) { + TEST_SKIP("`sasl.oauthbearer.method=OIDC` is required\n"); + rd_kafka_conf_destroy(conf); + return 0; + } + + test_timeout_set(300); + do_test_produce_consumer_with_OIDC("client_credentials", conf); + do_test_produce_share_consumer_with_OIDC("client_credentials", conf); + do_test_produce_consumer_with_OIDC_should_fail_invalid_token_endpoint( + conf); + do_test_produce_share_consumer_with_OIDC_should_fail_invalid_token_endpoint( + conf); + do_test_produce_consumer_with_OIDC_expired_token_should_fail(conf); + do_test_produce_share_consumer_with_OIDC_expired_token_should_fail( + conf); + do_test_produce_consumer_with_OIDC_jwt_bearer(conf); + do_test_produce_share_consumer_with_OIDC_jwt_bearer(conf); + do_test_produce_consumer_with_OIDC_metadata_authentication(conf); + do_test_produce_share_consumer_with_OIDC_metadata_authentication(conf); + do_test_produce_consumer_with_OIDC_sub_claim(conf); + do_test_produce_share_consumer_with_OIDC_sub_claim(conf); + + do_test_OIDC_aws_iam_conf_roundtrip(); + do_test_OIDC_aws_iam_stub_fires_without_callback(conf); + do_test_OIDC_aws_iam_user_callback_wins(conf); + + rd_kafka_conf_destroy(conf); + + return 0; +} diff --git a/lib/librdkafka-2.10.1/tests/0127-fetch_queue_backoff.cpp b/lib/librdkafka-2.15.0/tests/0127-fetch_queue_backoff.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0127-fetch_queue_backoff.cpp rename to lib/librdkafka-2.15.0/tests/0127-fetch_queue_backoff.cpp diff --git a/lib/librdkafka-2.10.1/tests/0128-sasl_callback_queue.cpp b/lib/librdkafka-2.15.0/tests/0128-sasl_callback_queue.cpp similarity index 55% rename from lib/librdkafka-2.10.1/tests/0128-sasl_callback_queue.cpp rename to lib/librdkafka-2.15.0/tests/0128-sasl_callback_queue.cpp index aaf23a081bf..8931f42568d 100644 --- a/lib/librdkafka-2.10.1/tests/0128-sasl_callback_queue.cpp +++ b/lib/librdkafka-2.15.0/tests/0128-sasl_callback_queue.cpp @@ -34,6 +34,11 @@ #include "testcpp.h" #include "rdatomic.h" +/* Include C API for share consumer tests */ +extern "C" { +#include "rdkafka.h" +} + namespace { /* Provide our own token refresh callback */ class MyCb : public RdKafka::OAuthBearerTokenRefreshCb { @@ -110,6 +115,91 @@ static void do_test(bool use_background_queue) { SUB_TEST_PASS(); } +static rd_atomic32_t share_cb_called; + +static void share_refresh_cb(rd_kafka_t *rk, + const char *oauthbearer_config, + void *opaque) { + rd_kafka_oauthbearer_set_token_failure(rk, + "Not implemented by this test, " + "but that's okay"); + rd_atomic32_add(&share_cb_called, 1); + Test::Say("Share consumer refresh callback called!\n"); +} + +/** + * @brief Verify that background SASL callback queues work with + * a share consumer using the C API. + * + * When use_background_queue is true, the SASL queue is enabled and + * forwarded to the background thread. The callback should fire even + * from a non-polling API (clusterid). + * + * When use_background_queue is false, the callback should still fire + * when polling via share_consume_batch (which serves the main queue). + */ +static void do_test_share_consumer(bool use_background_queue) { + SUB_TEST("Share consumer: Use background queue = %s", + use_background_queue ? "yes" : "no"); + + rd_kafka_conf_t *conf = rd_kafka_conf_new(); + char errstr[512]; + + rd_kafka_conf_set(conf, "security.protocol", "SASL_PLAINTEXT", errstr, + sizeof(errstr)); + rd_kafka_conf_set(conf, "sasl.mechanism", "OAUTHBEARER", errstr, + sizeof(errstr)); + rd_kafka_conf_set(conf, "group.id", "share-sasl-callback-test", errstr, + sizeof(errstr)); + + if (use_background_queue) + rd_kafka_conf_enable_sasl_queue(conf, 1); + + rd_atomic32_init(&share_cb_called, 0); + rd_kafka_conf_set_oauthbearer_token_refresh_cb(conf, share_refresh_cb); + + rd_kafka_share_t *rkshare = + rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer: %s", errstr); + + if (use_background_queue) { + rd_kafka_error_t *error = + rd_kafka_share_sasl_background_callbacks_enable(rkshare); + if (error) + Test::Fail("share_sasl_background_callbacks_enable() failed: " + + std::string(rd_kafka_error_string(error))); + + /* Call a non-polling share consumer API — the callback should + * still fire via the background thread. */ + rd_kafka_topic_partition_list_t *sub = NULL; + rd_kafka_share_subscription(rkshare, &sub); + if (sub) + rd_kafka_topic_partition_list_destroy(sub); + + } else { + /* Poll via share_poll — this serves the main queue + * which should trigger the OAUTHBEARER refresh callback. */ + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err = rd_kafka_share_poll(rkshare, 1000, &batch); + if (err) + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + Test::Say(tostr() << "share_cb_called = " << rd_atomic32_get(&share_cb_called) + << "\n"); + TEST_ASSERT(rd_atomic32_get(&share_cb_called) > 0, + "Expected refresh callback to have been called " + "for share consumer"); + + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + extern "C" { int main_0128_sasl_callback_queue(int argc, char **argv) { if (!test_check_builtin("sasl_oauthbearer")) { @@ -119,6 +209,8 @@ int main_0128_sasl_callback_queue(int argc, char **argv) { do_test(true); do_test(false); + do_test_share_consumer(true); + do_test_share_consumer(false); return 0; } diff --git a/lib/librdkafka-2.10.1/tests/0129-fetch_aborted_msgs.c b/lib/librdkafka-2.15.0/tests/0129-fetch_aborted_msgs.c similarity index 99% rename from lib/librdkafka-2.10.1/tests/0129-fetch_aborted_msgs.c rename to lib/librdkafka-2.15.0/tests/0129-fetch_aborted_msgs.c index 5d9b63b74f5..0206f7adf1f 100644 --- a/lib/librdkafka-2.10.1/tests/0129-fetch_aborted_msgs.c +++ b/lib/librdkafka-2.15.0/tests/0129-fetch_aborted_msgs.c @@ -58,7 +58,7 @@ int main_0129_fetch_aborted_msgs(int argc, char **argv) { test_admin_create_topic(rk, topic, 1, 1, (const char *[]) {"max.message.bytes", "10000", - "segment.bytes", "20000", + "segment.bytes", "1048576", NULL}); test_wait_topic_exists(rk, topic, 5000); diff --git a/lib/librdkafka-2.10.1/tests/0130-store_offsets.c b/lib/librdkafka-2.15.0/tests/0130-store_offsets.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0130-store_offsets.c rename to lib/librdkafka-2.15.0/tests/0130-store_offsets.c diff --git a/lib/librdkafka-2.10.1/tests/0131-connect_timeout.c b/lib/librdkafka-2.15.0/tests/0131-connect_timeout.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0131-connect_timeout.c rename to lib/librdkafka-2.15.0/tests/0131-connect_timeout.c diff --git a/lib/librdkafka-2.10.1/tests/0132-strategy_ordering.c b/lib/librdkafka-2.15.0/tests/0132-strategy_ordering.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0132-strategy_ordering.c rename to lib/librdkafka-2.15.0/tests/0132-strategy_ordering.c diff --git a/lib/librdkafka-2.10.1/tests/0133-ssl_keys.c b/lib/librdkafka-2.15.0/tests/0133-ssl_keys.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0133-ssl_keys.c rename to lib/librdkafka-2.15.0/tests/0133-ssl_keys.c diff --git a/lib/librdkafka-2.10.1/tests/0134-ssl_provider.c b/lib/librdkafka-2.15.0/tests/0134-ssl_provider.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0134-ssl_provider.c rename to lib/librdkafka-2.15.0/tests/0134-ssl_provider.c diff --git a/lib/librdkafka-2.15.0/tests/0135-sasl_credentials.cpp b/lib/librdkafka-2.15.0/tests/0135-sasl_credentials.cpp new file mode 100644 index 00000000000..72e3e7ec13b --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0135-sasl_credentials.cpp @@ -0,0 +1,279 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2022, Magnus Edenhill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +/** + * Verify that SASL credentials can be updated. + */ +#include "testcpp.h" + +/* Include C API and test helpers for share consumer tests */ +extern "C" { +#include "test.h" +#include "rdkafka.h" +} + + + +class authErrorEventCb : public RdKafka::EventCb { + public: + authErrorEventCb() : error_seen(false) { + } + + void event_cb(RdKafka::Event &event) { + switch (event.type()) { + case RdKafka::Event::EVENT_ERROR: + Test::Say(tostr() << "Error: " << RdKafka::err2str(event.err()) << ": " + << event.str() << "\n"); + if (event.err() == RdKafka::ERR__AUTHENTICATION) + error_seen = true; + break; + + case RdKafka::Event::EVENT_LOG: + Test::Say(tostr() << "Log: " << event.str() << "\n"); + break; + + default: + break; + } + } + + bool error_seen; +}; + + +/** + * @brief Test setting SASL credentials. + * + * 1. Switch out the proper username/password for invalid ones. + * 2. Verify that we get an auth failure. + * 3. Set the proper username/passwords. + * 4. Verify that we can now connect. + */ +static void do_test(bool set_after_auth_failure) { + RdKafka::Conf *conf; + Test::conf_init(&conf, NULL, 30); + + SUB_TEST_QUICK("set_after_auth_failure=%s", + set_after_auth_failure ? "yes" : "no"); + + /* Get the correct sasl.username and sasl.password */ + std::string username, password; + if (conf->get("sasl.username", username) || + conf->get("sasl.password", password)) { + delete conf; + SUB_TEST_SKIP("sasl.username and/or sasl.password not configured\n"); + return; + } + + /* Replace with incorrect ones */ + Test::conf_set(conf, "sasl.username", "ThisIsNotRight"); + Test::conf_set(conf, "sasl.password", "Neither Is This"); + + /* Set up an event callback to track authentication errors */ + authErrorEventCb pEvent = authErrorEventCb(); + std::string errstr; + if (conf->set("event_cb", &pEvent, errstr) != RdKafka::Conf::CONF_OK) + Test::Fail(errstr); + + /* Create client */ + RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr); + if (!p) + Test::Fail("Failed to create Producer: " + errstr); + delete conf; + + if (set_after_auth_failure) { + Test::Say("Awaiting auth failure\n"); + + while (!pEvent.error_seen) + p->poll(1000); + + Test::Say("Authentication error seen\n"); + } + + Test::Say("Setting proper credentials\n"); + RdKafka::Error *error = p->sasl_set_credentials(username, password); + if (error) + Test::Fail("Failed to set credentials: " + error->str()); + + Test::Say("Expecting successful cluster authentication\n"); + const std::string clusterid = p->clusterid(5 * 1000); + + if (clusterid.empty()) + Test::Fail("Expected clusterid() to succeed"); + + delete p; + + SUB_TEST_PASS(); +} + +static rd_bool_t share_error_seen = rd_false; + +static void share_auth_error_cb(rd_kafka_t *rk, + int err, + const char *reason, + void *opaque) { + if (err == RD_KAFKA_RESP_ERR__AUTHENTICATION) { + Test::Say(tostr() << "Share consumer auth error: " + << rd_kafka_err2str((rd_kafka_resp_err_t)err) << ": " + << reason << "\n"); + share_error_seen = rd_true; + } +} + +/** + * @brief Test setting SASL credentials on a share consumer. + * + * 1. Produce a message to a topic. + * 2. Create share consumer with wrong credentials (or correct if + * set_after_auth_failure is false). + * 3. Optionally wait for auth failure. + * 4. Set correct credentials via rd_kafka_share_sasl_set_credentials(). + * 5. Verify share consumer can consume the message. + */ +static void do_test_share_consumer(bool set_after_auth_failure) { + rd_kafka_conf_t *conf; + rd_kafka_t *p1; + rd_kafka_share_t *sc; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + char errstr[512]; + char *username, *password; + const char *topic; + const char *group = "share-sasl-creds-test"; + size_t rcvd; + int attempts; + + SUB_TEST("Share consumer: set_after_auth_failure=%s", + set_after_auth_failure ? "yes" : "no"); + + /* Get correct credentials from test config */ + test_conf_init(&conf, NULL, 30); + username = rd_strdup(test_conf_get(conf, "sasl.username")); + password = rd_strdup(test_conf_get(conf, "sasl.password")); + + /* Create a producer with correct creds, produce a message first */ + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + p1 = test_create_handle(RD_KAFKA_PRODUCER, conf); + + topic = test_mk_topic_name("0135_share_sasl_creds", 1); + test_create_topic_wait_exists(p1, topic, 1, 3, 5000); + + /* Set group config for earliest offset */ + test_share_set_auto_offset_reset(group, "earliest"); + + /* Produce a message */ + test_produce_msgs_simple(p1, topic, 0, 1); + + /* Create share consumer */ + test_conf_init(&conf, NULL, 30); + if (set_after_auth_failure) { + /* Start with wrong credentials */ + test_conf_set(conf, "sasl.username", "ThisIsNotRight"); + test_conf_set(conf, "sasl.password", "Neither Is This"); + } + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + + share_error_seen = rd_false; + rd_kafka_conf_set_error_cb(conf, share_auth_error_cb); + + sc = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(sc != NULL, "Failed to create share consumer: %s", errstr); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc, subs); + rd_kafka_topic_partition_list_destroy(subs); + + if (set_after_auth_failure) { + Test::Say("Awaiting share consumer auth failure\n"); + while (!share_error_seen) { + err = rd_kafka_share_poll(sc, 1000, &batch); + if (err) + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + Test::Say("Share consumer authentication error seen\n"); + } + + /* Set correct credentials */ + Test::Say("Setting proper credentials on share consumer\n"); + err = rd_kafka_share_sasl_set_credentials(sc, username, password); + TEST_ASSERT(!err, "share_sasl_set_credentials failed: %s", + err ? rd_kafka_error_string(err) : ""); + + /* Consume the message to verify connectivity */ + attempts = 100; + rcvd = 0; + while (rcvd == 0 && attempts-- > 0) { + err = rd_kafka_share_poll(sc, 3000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + if (rcvd == 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + TEST_ASSERT(rcvd > 0, + "Share consumer failed to consume message after " + "credential update"); + rd_kafka_messages_destroy(batch); + batch = NULL; + + Test::Say("Share consumer successfully consumed after credential update\n"); + + rd_kafka_share_consumer_close(sc); + rd_kafka_share_destroy(sc); + rd_kafka_destroy(p1); + free(username); + free(password); + + SUB_TEST_PASS(); +} + +extern "C" { +int main_0135_sasl_credentials(int argc, char **argv) { + const char *mech = test_conf_get(NULL, "sasl.mechanism"); + + if (strcmp(mech, "PLAIN") && strncmp(mech, "SCRAM", 5)) { + Test::Skip("Test requires SASL PLAIN or SASL SCRAM\n"); + return 0; + } + + do_test(false); + do_test(true); + do_test_share_consumer(false); + do_test_share_consumer(true); + + return 0; +} +} diff --git a/lib/librdkafka-2.10.1/tests/0136-resolve_cb.c b/lib/librdkafka-2.15.0/tests/0136-resolve_cb.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0136-resolve_cb.c rename to lib/librdkafka-2.15.0/tests/0136-resolve_cb.c diff --git a/lib/librdkafka-2.10.1/tests/0137-barrier_batch_consume.c b/lib/librdkafka-2.15.0/tests/0137-barrier_batch_consume.c similarity index 80% rename from lib/librdkafka-2.10.1/tests/0137-barrier_batch_consume.c rename to lib/librdkafka-2.15.0/tests/0137-barrier_batch_consume.c index 19bec387db6..18f592ddf68 100644 --- a/lib/librdkafka-2.10.1/tests/0137-barrier_batch_consume.c +++ b/lib/librdkafka-2.15.0/tests/0137-barrier_batch_consume.c @@ -608,12 +608,145 @@ static void do_test_consume_batch_control_msgs(void) { } +/** + * @brief Test that rd_kafka_consume_batch_queue correctly updates consumer + * position when EOF messages are received with + * enable.partition.eof=true. + * + * This is a regression test for the bug where EOF messages incorrectly + * advanced the consumer position by 2 instead of 1 (last_offset + 2 instead + * of last_offset + 1). + */ +static void do_test_consume_batch_eof_position(void) { + const char *topic; + rd_kafka_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_queue_t *rkq; + uint64_t testid; + const int partition_cnt = 1; + const int partition = 0; + const int produce_msg_cnt = 5; + const int consume_msg_cnt = 10; + const int timeout_ms = 1000; + const int session_timeout_s = 60; + const int replication_factor = -1; + const int topic_creation_timeout_ms = 5000; + rd_kafka_message_t **rkmessages; + int msg_cnt, i; + int64_t last_real_offset = -1; + int64_t eof_offset = -1; + int64_t position_after_eof; + rd_kafka_topic_partition_list_t *positions; + rd_kafka_resp_err_t err; + int eof_received = 0; + + SUB_TEST_QUICK("Testing EOF position with consume_batch_queue"); + + /* Create consumer configuration with enable.partition.eof=true */ + test_conf_init(&conf, NULL, session_timeout_s); + test_conf_set(conf, "enable.auto.commit", "false"); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "enable.partition.eof", "true"); + + testid = test_id_generate(); + + topic = test_mk_topic_name("0137-barrier_batch_consume", 1); + + /* Create topic */ + test_create_topic_wait_exists(NULL, topic, partition_cnt, + replication_factor, + topic_creation_timeout_ms); + test_produce_msgs_easy(topic, testid, partition, produce_msg_cnt); + + consumer = test_create_consumer(topic, NULL, conf, NULL); + test_consumer_subscribe(consumer, topic); + test_consumer_wait_assignment(consumer, rd_false); + + /* Create generic consume queue */ + rkq = rd_kafka_queue_get_consumer(consumer); + + /* Consume messages in batches until we get EOF */ + rkmessages = malloc(consume_msg_cnt * sizeof(*rkmessages)); + + while (!eof_received) { + msg_cnt = (int)rd_kafka_consume_batch_queue( + rkq, timeout_ms, rkmessages, consume_msg_cnt); + + TEST_ASSERT(msg_cnt >= 0, "consume_batch_queue failed"); + + if (msg_cnt == 0) { + TEST_WARN("No messages received, retrying..."); + continue; + } + + /* Check if EOF messages are received */ + for (i = 0; i < msg_cnt; i++) { + rd_kafka_message_t *rkm = rkmessages[i]; + + if (rkm->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) { + eof_received = 1; + eof_offset = rkm->offset; + TEST_SAY("Received EOF at offset %" PRId64 "\n", + eof_offset); + } else if (!rkm->err) { + last_real_offset = rkm->offset; + } + } + + /* Destroy messages */ + for (i = 0; i < msg_cnt; i++) + rd_kafka_message_destroy(rkmessages[i]); + } + + rd_free(rkmessages); + + /* Test that the last real message offset is the expected value */ + TEST_ASSERT(last_real_offset == produce_msg_cnt - 1, + "Expected last message offset %" PRId64 ", got %" PRId64, + (int64_t)(produce_msg_cnt - 1), last_real_offset); + + /* Get consumer position after EOF */ + positions = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(positions, topic, partition); + + err = rd_kafka_position(consumer, positions); + TEST_ASSERT(!err, "rd_kafka_position failed: %s", + rd_kafka_err2str(err)); + + /* Extract the position value from the partition list */ + position_after_eof = positions->elems[0].offset; + + TEST_SAY("Last real message offset: %" PRId64 + "\n" + "EOF offset: %" PRId64 + "\n" + "Position after EOF: %" PRId64 "\n", + last_real_offset, eof_offset, position_after_eof); + + TEST_ASSERT(position_after_eof == last_real_offset + 1, + "Position after EOF should be %" PRId64 + " (last_offset + 1), " + "but got %" PRId64 + ". This indicates the EOF offset bug where " + "position incorrectly advances by +2 instead of +1", + last_real_offset + 1, position_after_eof); + + rd_kafka_topic_partition_list_destroy(positions); + rd_kafka_queue_destroy(rkq); + test_consumer_close(consumer); + rd_kafka_destroy(consumer); + + SUB_TEST_PASS(); +} + + int main_0137_barrier_batch_consume(int argc, char **argv) { do_test_consume_batch_with_seek(); do_test_consume_batch_store_offset(); do_test_consume_batch_with_pause_and_resume_different_batch(); do_test_consume_batch_with_pause_and_resume_same_batch(); do_test_consume_batch_control_msgs(); + do_test_consume_batch_eof_position(); return 0; } diff --git a/lib/librdkafka-2.10.1/tests/0138-admin_mock.c b/lib/librdkafka-2.15.0/tests/0138-admin_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0138-admin_mock.c rename to lib/librdkafka-2.15.0/tests/0138-admin_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0139-offset_validation_mock.c b/lib/librdkafka-2.15.0/tests/0139-offset_validation_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0139-offset_validation_mock.c rename to lib/librdkafka-2.15.0/tests/0139-offset_validation_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0140-commit_metadata.cpp b/lib/librdkafka-2.15.0/tests/0140-commit_metadata.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/0140-commit_metadata.cpp rename to lib/librdkafka-2.15.0/tests/0140-commit_metadata.cpp diff --git a/lib/librdkafka-2.10.1/tests/0142-reauthentication.c b/lib/librdkafka-2.15.0/tests/0142-reauthentication.c similarity index 50% rename from lib/librdkafka-2.10.1/tests/0142-reauthentication.c rename to lib/librdkafka-2.15.0/tests/0142-reauthentication.c index c6f3f4b4f35..98af27a16e2 100644 --- a/lib/librdkafka-2.10.1/tests/0142-reauthentication.c +++ b/lib/librdkafka-2.15.0/tests/0142-reauthentication.c @@ -111,6 +111,135 @@ void do_test_producer(int64_t reauth_time, const char *topic) { SUB_TEST_PASS(); } +/* Test share consumer message loss while reauth happens between consume. */ +void do_test_share_consumer(int64_t reauth_time) { + rd_kafka_t *p1; + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + const char *grp_conf[] = {"share.auto.offset.reset", "SET", "earliest"}; + const char *group = "share-reauth-test"; + const char *topic = test_mk_topic_name("share_reauth", 1); + int64_t start_time = 0; + int64_t wait_time = reauth_time * 1.2 * 1000; + int recv_cnt = 0, sent_cnt = 0; + int attempts; + size_t rcvd, m; + + SUB_TEST("test share consumer message loss while reauthenticating"); + + test_conf_init(&conf, NULL, 30); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + p1 = test_create_handle(RD_KAFKA_PRODUCER, conf); + + test_create_topic_wait_exists(p1, topic, 1, 3, 5000); + TEST_SAY("Topic: %s is created\n", topic); + + /* Create share consumer */ + sc1 = test_create_share_consumer(group, NULL); + + /* Set group config for earliest offset */ + test_IncrementalAlterConfigs_simple(p1, RD_KAFKA_RESOURCE_GROUP, group, + grp_conf, 1); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Wait for the share consumer to join the group by producing + * a message and polling until it is received. */ + test_produce_msgs2(p1, topic, 0, 0, 0, 1, NULL, 0); + rd_kafka_flush(p1, 10 * 1000); + attempts = 50; + rcvd = 0; + while (rcvd == 0 && attempts-- > 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(sc1, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + } + TEST_ASSERT(rcvd > 0, + "Share consumer failed to join group " + "and consume warmup message"); + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_SAY("Share consumer joined group, starting reauth test\n"); + + start_time = test_clock(); + while ((test_clock() - start_time) <= wait_time) { + rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Produce one message. */ + test_produce_msgs2(p1, topic, 0, 0, 0, 1, NULL, 0); + sent_cnt++; + + err = rd_kafka_share_poll(sc1, 100, &batch); + if (err) { + rd_kafka_error_destroy(err); + rd_kafka_flush(p1, 50); + } else { + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, m); + if (!rkm->err) + recv_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Maintain the message rate at ~200 msg/s regardless + * of consume errors during reauthentication. */ + rd_usleep(1000 * 50, NULL); + } + + /* Final flush and receive any remaining messages. */ + rd_kafka_flush(p1, 10 * 1000); + attempts = 100; + while (recv_cnt < sent_cnt && attempts-- > 0) { + rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + + err = rd_kafka_share_poll(sc1, 3000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, m); + if (!rkm->err) + recv_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + rd_kafka_share_consumer_close(sc1); + + TEST_ASSERT(sent_cnt == recv_cnt, + "did not receive as many messages as sent (%d vs %d)", + sent_cnt, recv_cnt); + + rd_kafka_destroy(p1); + rd_kafka_share_destroy(sc1); + SUB_TEST_PASS(); +} + + /* Test consumer message loss while reauth happens between consume. */ void do_test_consumer(int64_t reauth_time, const char *topic) { uint64_t testid; @@ -261,6 +390,182 @@ void do_test_txn_producer(int64_t reauth_time, /* Check reauthentication in case of OAUTHBEARER mechanism, with different * reauth times and token lifetimes. */ +void do_test_share_oauthbearer(int64_t reauth_time, + int64_t token_lifetime_ms, + rd_bool_t use_sasl_queue) { + rd_kafka_t *p1; + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + const char *grp_conf[] = {"share.auto.offset.reset", "SET", "earliest"}; + const char *group = "share-oauthbearer-reauth-test"; + const char *topic = test_mk_topic_name("share_oauth_reauth", 1); + char *mechanism, *oauthbearer_method; + char errstr[512]; + int token_lifetime_s = token_lifetime_ms / 1000; + int64_t start_time = 0; + int64_t wait_time = reauth_time * 1.2 * 1000; + int recv_cnt = 0, sent_cnt = 0; + int attempts; + size_t rcvd, m; + + SUB_TEST( + "test share consumer reauthentication with oauthbearer, " + "reauth_time = %ld, token_lifetime = %ld, use_sasl_queue = %s", + reauth_time, token_lifetime_ms, use_sasl_queue ? "yes" : "no"); + + /* Producer */ + test_conf_init(&conf, NULL, 30); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + mechanism = test_conf_get(conf, "sasl.mechanism"); + if (rd_strcasecmp(mechanism, "oauthbearer")) { + rd_kafka_conf_destroy(conf); + SUB_TEST_SKIP( + "`sasl.mechanism=OAUTHBEARER` is required, have %s\n", + mechanism); + } + + oauthbearer_method = test_conf_get(conf, "sasl.oauthbearer.method"); + if (rd_strcasecmp(oauthbearer_method, "oidc")) { + test_conf_set( + conf, "sasl.oauthbearer.config", + tsprintf( + "principal=admin scope=requiredScope lifeSeconds=%d", + token_lifetime_s)); + test_conf_set(conf, "enable.sasl.oauthbearer.unsecure.jwt", + "true"); + } + + p1 = test_create_handle(RD_KAFKA_PRODUCER, conf); + test_create_topic_wait_exists(p1, topic, 1, 3, 5000); + + /* Share consumer */ + test_conf_init(&conf, NULL, 30); + rd_kafka_conf_enable_sasl_queue(conf, use_sasl_queue); + + oauthbearer_method = test_conf_get(conf, "sasl.oauthbearer.method"); + if (rd_strcasecmp(oauthbearer_method, "oidc")) { + test_conf_set( + conf, "sasl.oauthbearer.config", + tsprintf( + "principal=admin scope=requiredScope lifeSeconds=%d", + token_lifetime_s)); + test_conf_set(conf, "enable.sasl.oauthbearer.unsecure.jwt", + "true"); + } + + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + sc1 = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(sc1 != NULL, "Failed to create share consumer: %s", errstr); + + if (use_sasl_queue) { + rd_kafka_error_t *error = + rd_kafka_share_sasl_background_callbacks_enable(sc1); + TEST_ASSERT(!error, + "share_sasl_background_callbacks_enable failed: %s", + error ? rd_kafka_error_string(error) : ""); + } + + /* Set group config for earliest offset */ + test_IncrementalAlterConfigs_simple(p1, RD_KAFKA_RESOURCE_GROUP, group, + grp_conf, 1); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Wait for the share consumer to join the group by producing + * a message and polling until it is received. */ + test_produce_msgs2(p1, topic, 0, 0, 0, 1, NULL, 0); + rd_kafka_flush(p1, 10 * 1000); + attempts = 50; + rcvd = 0; + while (rcvd == 0 && attempts-- > 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(sc1, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + } + TEST_ASSERT(rcvd > 0, + "Share consumer failed to join group " + "and consume warmup message"); + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_SAY( + "Share consumer joined group, starting oauthbearer " + "reauth test\n"); + + start_time = test_clock(); + while ((test_clock() - start_time) <= wait_time) { + rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_produce_msgs2(p1, topic, 0, 0, 0, 1, NULL, 0); + sent_cnt++; + + err = rd_kafka_share_poll(sc1, 100, &batch); + if (err) { + rd_kafka_error_destroy(err); + rd_kafka_flush(p1, 50); + } else { + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, m); + if (!rkm->err) + recv_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + rd_usleep(1000 * 50, NULL); + } + + /* Final flush and receive any remaining messages. */ + rd_kafka_flush(p1, 10 * 1000); + attempts = 100; + while (recv_cnt < sent_cnt && attempts-- > 0) { + rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + + err = rd_kafka_share_poll(sc1, 3000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, m); + if (!rkm->err) + recv_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + rd_kafka_share_consumer_close(sc1); + + TEST_ASSERT(sent_cnt == recv_cnt, + "did not receive as many messages as sent (%d vs %d)", + sent_cnt, recv_cnt); + + rd_kafka_destroy(p1); + rd_kafka_share_destroy(sc1); + SUB_TEST_PASS(); +} + + void do_test_oauthbearer(int64_t reauth_time, const char *topic, int64_t token_lifetime_ms, @@ -270,7 +575,7 @@ void do_test_oauthbearer(int64_t reauth_time, rd_kafka_t *rk = NULL; uint64_t testid = test_id_generate(); rd_kafka_resp_err_t err; - char *mechanism; + char *mechanism, *oauthbearer_method; int msgrate, msgcnt, sent_msg; test_timing_t t_produce; int token_lifetime_s = token_lifetime_ms / 1000; @@ -292,11 +597,16 @@ void do_test_oauthbearer(int64_t reauth_time, mechanism); } - test_conf_set( - conf, "sasl.oauthbearer.config", - tsprintf("principal=admin scope=requiredScope lifeSeconds=%d", - token_lifetime_s)); - test_conf_set(conf, "enable.sasl.oauthbearer.unsecure.jwt", "true"); + oauthbearer_method = test_conf_get(conf, "sasl.oauthbearer.method"); + if (rd_strcasecmp(oauthbearer_method, "oidc")) { + test_conf_set( + conf, "sasl.oauthbearer.config", + tsprintf( + "principal=admin scope=requiredScope lifeSeconds=%d", + token_lifetime_s)); + test_conf_set(conf, "enable.sasl.oauthbearer.unsecure.jwt", + "true"); + } rk = test_create_handle(RD_KAFKA_PRODUCER, conf); /* Enable to background queue since we don't want to poll the SASL @@ -403,11 +713,116 @@ void do_test_reauth_failure(int64_t reauth_time, const char *topic) { } +/* Test share consumer reauth failure with wrong PLAIN/SCRAM credentials. */ +void do_test_share_reauth_failure(int64_t reauth_time) { + rd_kafka_t *p1; + rd_kafka_share_t *sc1; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + const char *group = "share-reauth-failure-test"; + const char *topic = test_mk_topic_name("share_reauth_fail", 1); + char *mechanism; + char errstr[512]; + int64_t start_time = 0; + int64_t wait_time = reauth_time * 1.2 * 1000; + int attempts; + size_t rcvd; + + error_seen = 0; + expect_err = 0; + + SUB_TEST("test share consumer reauth failure with wrong credentials"); + + test_conf_init(&conf, NULL, 30); + mechanism = test_conf_get(conf, "sasl.mechanism"); + if (!rd_strcasecmp(mechanism, "oauthbearer")) { + rd_kafka_conf_destroy(conf); + SUB_TEST_SKIP( + "PLAIN or SCRAM mechanism is required, have " + "OAUTHBEARER"); + } + + /* Create producer with correct credentials */ + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + p1 = test_create_handle(RD_KAFKA_PRODUCER, conf); + test_create_topic_wait_exists(p1, topic, 1, 3, 5000); + + /* Create share consumer with correct credentials + error callback */ + test_conf_init(&conf, NULL, 30); + rd_kafka_conf_set_error_cb(conf, auth_error_cb); + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + sc1 = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(sc1 != NULL, "Failed to create share consumer: %s", errstr); + + /* Set group config for earliest offset */ + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(sc1, subs); + rd_kafka_topic_partition_list_destroy(subs); + + test_produce_msgs_simple(p1, topic, 0, 1); + attempts = 50; + rcvd = 0; + while (rcvd == 0 && attempts-- > 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(sc1, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + } + TEST_ASSERT(rcvd > 0, + "Share consumer failed to consume warmup message"); + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_SAY("Share consumer connected, producing messages\n"); + + test_produce_msgs_simple(p1, topic, 0, 200); + TEST_SAY("Produced messages\n"); + + /* Corrupt credentials */ + TEST_SAY("Corrupting credentials\n"); + rd_kafka_share_sasl_set_credentials(sc1, "somethingwhich", + "isnotright"); + expect_err = 1; + + /* Consume for 1.2x reauth interval. + * Reauth happens during consumption and fails with wrong creds. */ + start_time = test_clock(); + while ((test_clock() - start_time) <= wait_time) { + rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + + err = rd_kafka_share_poll(sc1, 100, &batch); + if (err) { + rd_kafka_error_destroy(err); + } else { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + + TEST_ASSERT(error_seen, "should have had an authentication error"); + + rd_kafka_share_consumer_close(sc1); + rd_kafka_destroy(p1); + rd_kafka_share_destroy(sc1); + SUB_TEST_PASS(); +} + + int main_0142_reauthentication(int argc, char **argv) { size_t broker_id_cnt; int32_t *broker_ids = NULL; rd_kafka_conf_t *conf = NULL; - const char *security_protocol, *sasl_mechanism; + const char *security_protocol; + rd_bool_t sasl_mechanism_oauthbearer = rd_false; + rd_bool_t oauthbearer_method_default = rd_false; size_t i; int64_t reauth_time = INT64_MAX; @@ -423,8 +838,11 @@ int main_0142_reauthentication(int argc, char **argv) { return 0; } - sasl_mechanism = test_conf_get(NULL, "sasl.mechanism"); - if (!rd_strcasecmp(sasl_mechanism, "oauthbearer")) + sasl_mechanism_oauthbearer = !rd_strcasecmp( + test_conf_get(NULL, "sasl.mechanism"), "oauthbearer"); + oauthbearer_method_default = !rd_strcasecmp( + test_conf_get(NULL, "sasl.oauthbearer.method"), "default"); + if (sasl_mechanism_oauthbearer && oauthbearer_method_default) test_conf_set(conf, "enable.sasl.oauthbearer.unsecure.jwt", "true"); @@ -467,12 +885,13 @@ int main_0142_reauthentication(int argc, char **argv) { return 0; } - /* Each test (7 of them) will take slightly more than 1 reauth_time + /* Each test (15 of them) will take slightly more than 1 reauth_time * interval. Additional 30s provide a reasonable buffer. */ - test_timeout_set(9 * reauth_time / 1000 + 30); + test_timeout_set(17 * reauth_time / 1000 + 30); do_test_consumer(reauth_time, topic); + do_test_share_consumer(reauth_time); do_test_producer(reauth_time, topic); do_test_txn_producer(reauth_time, topic, rd_false /* abort txn */); do_test_txn_producer(reauth_time, topic, rd_true /* abort txn */); @@ -485,15 +904,28 @@ int main_0142_reauthentication(int argc, char **argv) { * reauth time. */ do_test_oauthbearer(reauth_time, topic, reauth_time / 2, rd_true); do_test_oauthbearer(reauth_time, topic, reauth_time / 2, rd_false); - /* Case when the token_lifetime is greater than the maximum reauth time - * configured. - * In this case, the broker returns the maximum reauth time configured. - * We don't need to recreate the token, but we need to reauthenticate - * using the same token. */ - do_test_oauthbearer(reauth_time, topic, reauth_time * 2, rd_true); - do_test_oauthbearer(reauth_time, topic, reauth_time * 2, rd_false); + do_test_share_oauthbearer(reauth_time, reauth_time / 2, rd_true); + do_test_share_oauthbearer(reauth_time, reauth_time / 2, rd_false); + + /* With OIDC the expiration time is fixed so do the testing only + * once. */ + if (oauthbearer_method_default) { + /* Case when the token_lifetime is greater than the maximum + * reauth time configured. In this case, the broker returns the + * maximum reauth time configured. We don't need to recreate the + * token, but we need to reauthenticate using the same token. */ + do_test_oauthbearer(reauth_time, topic, reauth_time * 2, + rd_true); + do_test_oauthbearer(reauth_time, topic, reauth_time * 2, + rd_false); + do_test_share_oauthbearer(reauth_time, reauth_time * 2, + rd_true); + do_test_share_oauthbearer(reauth_time, reauth_time * 2, + rd_false); + } do_test_reauth_failure(reauth_time, topic); + do_test_share_reauth_failure(reauth_time); return 0; } diff --git a/lib/librdkafka-2.10.1/tests/0143-exponential_backoff_mock.c b/lib/librdkafka-2.15.0/tests/0143-exponential_backoff_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0143-exponential_backoff_mock.c rename to lib/librdkafka-2.15.0/tests/0143-exponential_backoff_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0144-idempotence_mock.c b/lib/librdkafka-2.15.0/tests/0144-idempotence_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0144-idempotence_mock.c rename to lib/librdkafka-2.15.0/tests/0144-idempotence_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0145-pause_resume_mock.c b/lib/librdkafka-2.15.0/tests/0145-pause_resume_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0145-pause_resume_mock.c rename to lib/librdkafka-2.15.0/tests/0145-pause_resume_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0146-metadata_mock.c b/lib/librdkafka-2.15.0/tests/0146-metadata_mock.c similarity index 99% rename from lib/librdkafka-2.10.1/tests/0146-metadata_mock.c rename to lib/librdkafka-2.15.0/tests/0146-metadata_mock.c index 7386125f259..50e8bd4dda7 100644 --- a/lib/librdkafka-2.10.1/tests/0146-metadata_mock.c +++ b/lib/librdkafka-2.15.0/tests/0146-metadata_mock.c @@ -190,9 +190,9 @@ static void do_test_fast_metadata_refresh(int variation) { mcluster, expected_metadata_requests, 500, is_metadata_request, NULL); TEST_ASSERT(expected_metadata_requests <= metadata_requests && - metadata_requests <= expected_metadata_requests + 1, + metadata_requests <= expected_metadata_requests + 2, "Expected %d or %d metadata request, got %d", - expected_metadata_requests, expected_metadata_requests + 1, + expected_metadata_requests, expected_metadata_requests + 2, metadata_requests); rd_kafka_mock_stop_request_tracking(mcluster); diff --git a/lib/librdkafka-2.15.0/tests/0147-consumer_group_consumer_mock.c b/lib/librdkafka-2.15.0/tests/0147-consumer_group_consumer_mock.c new file mode 100644 index 00000000000..3608863af46 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0147-consumer_group_consumer_mock.c @@ -0,0 +1,1210 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2024, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +#include + + +/** + * @name Mock tests specific of the KIP-848 group consumer protocol + */ + + +/** + * @enum test_variation_t + * @brief Variations for most error case tests. + */ +typedef enum test_variation_t { + /* Error happens on first HB */ + TEST_VARIATION_ERROR_FIRST_HB = 0, + /* Error happens on second HB */ + TEST_VARIATION_ERROR_SECOND_HB = 1, + TEST_VARIATION__CNT, +} test_variation_t; + +static const char *test_variation_name(test_variation_t variation) { + rd_assert(variation >= TEST_VARIATION_ERROR_FIRST_HB && + variation < TEST_VARIATION__CNT); + static const char *names[] = {"error on first heartbeat", + "error on second heartbeat"}; + return names[variation]; +} + +static int allowed_error; +static int rebalance_cnt; +static rd_kafka_resp_err_t rebalance_exp_event; +static rd_bool_t rebalance_exp_lost = rd_false; + +/** + * @brief Decide what error_cb's will cause the test to fail. + */ +static int +error_is_fatal_cb(rd_kafka_t *rk, rd_kafka_resp_err_t err, const char *reason) { + if (err == allowed_error || + /* If transport errors are allowed then it is likely + * that we'll also see ALL_BROKERS_DOWN. */ + (allowed_error == RD_KAFKA_RESP_ERR__TRANSPORT && + err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN)) { + TEST_SAY("Ignoring allowed error: %s: %s\n", + rd_kafka_err2name(err), reason); + return 0; + } + return 1; +} + +/** + * @brief Rebalance callback saving number of calls and verifying expected + * event. + */ +static void rebalance_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + rd_kafka_topic_partition_list_t *parts, + void *opaque) { + + rebalance_cnt++; + TEST_SAY("Rebalance #%d: %s: %d partition(s)\n", rebalance_cnt, + rd_kafka_err2name(err), parts->cnt); + + TEST_ASSERT( + err == rebalance_exp_event, "Expected rebalance event %s, not %s", + rd_kafka_err2name(rebalance_exp_event), rd_kafka_err2name(err)); + + if (rebalance_exp_lost) { + TEST_ASSERT(rd_kafka_assignment_lost(rk), + "Expected partitions lost"); + TEST_SAY("Partitions were lost\n"); + } + + test_rebalance_cb(rk, err, parts, opaque); + + rebalance_exp_event = RD_KAFKA_RESP_ERR_NO_ERROR; + /* Make sure only one rebalance callback is served per poll() + * so that expect_rebalance() returns to the test logic on each + * rebalance. */ + rd_kafka_yield(rk); +} + +static rd_bool_t is_heartbeat_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == + RD_KAFKAP_ConsumerGroupHeartbeat; +} + +/** + * @brief Wait at least \p num heartbeats + * have been received by the mock cluster + * plus \p confidence_interval has passed + * + * @return Number of heartbeats received. + */ +static int wait_all_heartbeats_done(rd_kafka_mock_cluster_t *mcluster, + int num, + int confidence_interval) { + return test_mock_wait_matching_requests( + mcluster, num, confidence_interval, is_heartbeat_request, NULL); +} + +static rd_kafka_t *create_consumer(const char *bootstraps, + const char *group_id, + rd_bool_t with_rebalance_cb) { + rd_kafka_conf_t *conf; + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "auto.offset.reset", "earliest"); + return test_create_consumer( + group_id, with_rebalance_cb ? rebalance_cb : NULL, conf, NULL); +} + +/** + * @brief Test heartbeat behavior with fatal errors, + * ensuring: + * - a fatal error is received on poll and consumer close + * - sequence: + * - in TEST_VARIATION_ERROR_FIRST_HB (1 HBs, 0 callbacks): + * - first HB returns a fatal error + * - no rebalance callbacks are called after that + * - all operations on the consumer fail with fatal error \p err + * - no final leave group HB is sent + * + * - in TEST_VARIATION_ERROR_SECOND_HB (2 HBs, 1 assignment callback): + * - first HB receives assignment + * - an assignment callback is called + * - second HB acknowledges the assignment and returns a fatal error. + * - no rebalance callbacks are called after that + * - all operations on the consumer fail with fatal error \p err. + * - no final leave group HB is sent + * + * @param err The error code to test. + * @param variation Test variation, see `test_variation_t`. + */ +static void +do_test_consumer_group_heartbeat_fatal_error(rd_kafka_resp_err_t err, + test_variation_t variation) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_t *c; + rd_kafka_message_t *rkmessage; + rd_kafka_resp_err_t fatal_error; + int expected_heartbeats, found_heartbeats, expected_rebalance_cnt; + test_timing_t timing; + rebalance_cnt = 0; + rebalance_exp_lost = rd_false; + rebalance_exp_event = RD_KAFKA_RESP_ERR_NO_ERROR; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + char errstr[512]; + + SUB_TEST_QUICK("%s, variation: %s", rd_kafka_err2name(err), + test_variation_name(variation)); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 1000); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + + TIMING_START(&timing, "consumer_group_heartbeat_fatal_error"); + + if (variation == TEST_VARIATION_ERROR_SECOND_HB) { + /* First HB returns assignment */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, 0); + } + + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, err, 0); + + c = create_consumer(bootstraps, topic, rd_true); + + /* Subscribe to the input topic */ + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + /* The partition is ignored in + * rd_kafka_subscribe() */ + RD_KAFKA_PARTITION_UA); + + TEST_SAY("Subscribing to topic\n"); + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_subscribe(c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + expected_heartbeats = 1; + + TEST_SAY("Awaiting all HBs\n"); + TEST_ASSERT((found_heartbeats = + wait_all_heartbeats_done(mcluster, expected_heartbeats, + 200)) == expected_heartbeats, + "Expected %d heartbeats, got %d", expected_heartbeats, + found_heartbeats); + + expected_rebalance_cnt = 0; + if (variation == TEST_VARIATION_ERROR_SECOND_HB) { + expected_rebalance_cnt++; + rebalance_exp_event = RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS; + + /* Trigger rebalance cb */ + rkmessage = rd_kafka_consumer_poll(c, 500); + TEST_ASSERT(!rkmessage, "No message should be returned"); + TEST_ASSERT( + rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected assign callback to be processed, but it wasn't"); + + /* Expect the acknowledge HB*/ + expected_heartbeats++; + TEST_ASSERT((found_heartbeats = wait_all_heartbeats_done( + mcluster, expected_heartbeats, 200)) == + expected_heartbeats, + "Expected %d heartbeats, got %d", + expected_heartbeats, found_heartbeats); + } + + rd_kafka_mock_clear_requests(mcluster); + TEST_SAY("Consume from c, a fatal error is returned\n"); + rkmessage = rd_kafka_consumer_poll(c, 500); + TEST_ASSERT(rkmessage != NULL, "An error message should be returned"); + TEST_ASSERT(rkmessage->err == RD_KAFKA_RESP_ERR__FATAL, + "Expected a _FATAL error, got %s", + rd_kafka_err2name(rkmessage->err)); + fatal_error = rd_kafka_fatal_error(c, errstr, sizeof(errstr)); + TEST_ASSERT(fatal_error == err, "Expected fatal error %s, got %s", + rd_kafka_err2name(err), rd_kafka_err2name(fatal_error)); + rd_kafka_message_destroy(rkmessage); + + TEST_ASSERT(rebalance_cnt == expected_rebalance_cnt, + "Expected %d rebalance events, got %d", + expected_rebalance_cnt, rebalance_cnt); + + /* Close c, a fatal error is returned */ + TEST_ASSERT(rd_kafka_consumer_close(c) == RD_KAFKA_RESP_ERR__FATAL, + "Expected a _FATAL error, got %s", rd_kafka_err2name(err)); + fatal_error = rd_kafka_fatal_error(c, errstr, sizeof(errstr)); + TEST_ASSERT(fatal_error == err, "Expected fatal error %s, got %s", + rd_kafka_err2name(err), rd_kafka_err2name(fatal_error)); + + TEST_ASSERT(rebalance_cnt == expected_rebalance_cnt, + "Expected %d rebalance events, got %d", + expected_rebalance_cnt, rebalance_cnt); + + rd_kafka_destroy(c); + + TEST_SAY("Ensuring there are no leave group HBs\n"); + TEST_ASSERT( + (found_heartbeats = wait_all_heartbeats_done(mcluster, 0, 0)) == 0, + "Expected no leave group heartbeat, got %d", found_heartbeats); + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + TIMING_ASSERT(&timing, 100, 1500); + SUB_TEST_PASS(); +} + +/** + * @brief Test all kind of fatal errors in a ConsumerGroupHeartbeat call. + * @sa test_variation_t + */ +static void do_test_consumer_group_heartbeat_fatal_errors(void) { + rd_kafka_resp_err_t fatal_errors[] = { + RD_KAFKA_RESP_ERR_INVALID_REQUEST, + RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED, + RD_KAFKA_RESP_ERR_UNSUPPORTED_ASSIGNOR, + RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION, + RD_KAFKA_RESP_ERR_UNRELEASED_INSTANCE_ID, + RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED, + RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND, + RD_KAFKA_RESP_ERR_CLUSTER_AUTHORIZATION_FAILED}; + size_t i; + test_variation_t j; + for (i = 0; i < RD_ARRAY_SIZE(fatal_errors); i++) { + /* Only these errors can happen on a second HB. */ + test_variation_t last_variation = + ((fatal_errors[i] == RD_KAFKA_RESP_ERR_INVALID_REQUEST) || + (fatal_errors[i] == + RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED)) + ? TEST_VARIATION_ERROR_SECOND_HB + : TEST_VARIATION_ERROR_FIRST_HB; + + for (j = TEST_VARIATION_ERROR_FIRST_HB; j <= last_variation; + j++) + do_test_consumer_group_heartbeat_fatal_error( + fatal_errors[i], j); + } +} + +/** + * @brief Test heartbeat behavior with retriable errors, + * ensuring: + * - no error is received on poll and consumer close + * - sequence: + * - in TEST_VARIATION_ERROR_FIRST_HB (4 HBs, 1 assignment callback, 1 + * revocation callback): + * - first HB is retried + * - second HB receives assignment + * - rebalance callback with an assignment + * - third HB for the acknowledgment + * - assignment revoked with a callback on consumer close + * - final leave group HB + * + * - in TEST_VARIATION_ERROR_SECOND_HB (4 HBs, 1 assignment callback, 1 + * revocation callback): + * - first HB receives assignment + * - assignment callback is called + * - second HB acknowledges the assignment and + * returns a retriable error + * - the HB is retried (third one) + * - assignment revoked with a callback on consumer close + * - final leave group HB + * + * @param err The error code to test. + * @param variation Test variation, see `test_variation_t`. + */ +static void +do_test_consumer_group_heartbeat_retriable_error(rd_kafka_resp_err_t err, + test_variation_t variation) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_t *c; + int expected_heartbeats, found_heartbeats; + test_timing_t timing; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_curr->is_fatal_cb = error_is_fatal_cb; + rebalance_cnt = 0; + rebalance_exp_lost = rd_false; + allowed_error = RD_KAFKA_RESP_ERR__TRANSPORT; + + SUB_TEST_QUICK("%s, variation: %s", rd_kafka_err2name(err), + test_variation_name(variation)); + + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 1000); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + + c = create_consumer(bootstraps, topic, rd_true); + + TIMING_START(&timing, "consumer_group_heartbeat_retriable_error"); + + if (variation == TEST_VARIATION_ERROR_SECOND_HB) { + /* First HB returns assignment */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, 0); + } + + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, err, 0); + + /* Subscribe to the input topic */ + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + /* The partition is ignored in + * rd_kafka_subscribe() */ + RD_KAFKA_PARTITION_UA); + + TEST_SAY("Subscribing to topic\n"); + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_subscribe(c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* TEST_VARIATION_ERROR_FIRST_HB First HB and its retry + ACK. */ + /* TEST_VARIATION_ERROR_SECOND_HB First HB + ACK and retry. */ + expected_heartbeats = 3; + rebalance_exp_event = RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS; + TEST_SAY( + "Consume from c, no message is returned, " + "but assign callback is processed\n"); + test_consumer_poll_no_msgs("after heartbeat", c, 0, 500); + TEST_ASSERT(rebalance_cnt > 0, "Expected > 0 rebalance events, got %d", + rebalance_cnt); + TEST_ASSERT(rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected assign callback to be processed, but it wasn't"); + + TEST_SAY("Awaiting first HBs\n"); + TEST_ASSERT((found_heartbeats = + wait_all_heartbeats_done(mcluster, expected_heartbeats, + 200)) == expected_heartbeats, + "Expected %d heartbeats, got %d", expected_heartbeats, + found_heartbeats); + + rebalance_exp_event = RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS; + + /* Close c without errors */ + expected_heartbeats++; + TEST_ASSERT(rd_kafka_consumer_close(c) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR, got %s", rd_kafka_err2name(err)); + TEST_ASSERT(rebalance_cnt > 0, "Expected > 0 rebalance events, got %d", + rebalance_cnt); + TEST_ASSERT(rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected revoke callback to be processed, but it wasn't"); + + rd_kafka_destroy(c); + + TEST_SAY("Awaiting leave group HB\n"); + TEST_ASSERT((found_heartbeats = + wait_all_heartbeats_done(mcluster, expected_heartbeats, + 0)) == expected_heartbeats, + "Expected %d heartbeats, got %d", expected_heartbeats, + found_heartbeats); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + TIMING_ASSERT(&timing, 100, 1500); + + test_curr->is_fatal_cb = NULL; + allowed_error = RD_KAFKA_RESP_ERR_NO_ERROR; + + SUB_TEST_PASS(); +} + +/** + * @brief Test all kind of retriable errors in a ConsumerGroupHeartbeat call. + * @sa test_variation_t + */ +static void do_test_consumer_group_heartbeat_retriable_errors(void) { + rd_kafka_resp_err_t retriable_errors[] = { + RD_KAFKA_RESP_ERR_COORDINATOR_LOAD_IN_PROGRESS, + RD_KAFKA_RESP_ERR__SSL, RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE}; + size_t i; + test_variation_t j; + for (i = 0; i < RD_ARRAY_SIZE(retriable_errors); i++) { + for (j = TEST_VARIATION_ERROR_FIRST_HB; j < TEST_VARIATION__CNT; + j++) + do_test_consumer_group_heartbeat_retriable_error( + retriable_errors[i], j); + } +} + +/** + * @brief Test heartbeat behavior with consumer fenced errors, + * ensuring: + * - no error is received on poll and consumer close + * - sequence: + * + * - in TEST_VARIATION_ERROR_FIRST_HB (4 HBs, 1 assignment callback, 1 + * revocation callback): + * - first HB fences the member + * it does not receives assignment + * or revoke any partitions + * - second HB receives assignment + * - there's an assignment callback + * - assignment is acknowledged (third HB) + * - assignment is revoked on close (fourth HB) + * - last revoke callback + * + * - in TEST_VARIATION_ERROR_SECOND_HB (5 HBs, 2 assignment callbacks, + * 2 revocation callbacks of which 1 as lost): + * - first HB receives assignment + * - assignment callback is called + * - second HB acknowledges the assignment, fences the consumer + * - a lost callback is called (lost partitions) + * - partitions are assigned again on re-joining (third HB) + * - second assignment callback + * - acknowledgment of the assignment (fourth HB) + * - partitions are revoked on close (fifth HB) + * - last revoke callback + * + * @param err The error code to test. + * @param variation Test variation, see `test_variation_t`. + */ +static void +do_test_consumer_group_heartbeat_fenced_error(rd_kafka_resp_err_t err, + test_variation_t variation) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_t *c; + rd_kafka_message_t *rkmessage; + int expected_heartbeats, found_heartbeats, expected_rebalance_cnt; + test_timing_t timing; + rebalance_cnt = 0; + rebalance_exp_lost = rd_false; + rebalance_exp_event = RD_KAFKA_RESP_ERR_NO_ERROR; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + + SUB_TEST_QUICK("%s, variation: %s", rd_kafka_err2name(err), + test_variation_name(variation)); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 1000); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + + if (variation == TEST_VARIATION_ERROR_SECOND_HB) { + /* First HB returns assignment */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, 0); + } + + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, err, 0); + + c = create_consumer(bootstraps, topic, rd_true); + + TIMING_START(&timing, "consumer_group_heartbeat_fenced_error"); + + /* Subscribe to the input topic */ + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + /* The partition is ignored in + * rd_kafka_subscribe() */ + RD_KAFKA_PARTITION_UA); + + TEST_SAY("Subscribing to topic\n"); + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_subscribe(c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* variation ERROR_FIRST_HB: First HB fences and second receives + * the assignment*/ + expected_heartbeats = 2; + if (variation == TEST_VARIATION_ERROR_SECOND_HB) + /* variation ERROR_SECOND_HB: First HB receives assignment, + * second HB fences the consumer. + * We only await one here as we need to process the assignment + * callback. */ + expected_heartbeats = 1; + + TEST_SAY("Awaiting initial HBs\n"); + TEST_ASSERT((found_heartbeats = + wait_all_heartbeats_done(mcluster, expected_heartbeats, + 200)) == expected_heartbeats, + "Expected %d heartbeats, got %d", expected_heartbeats, + found_heartbeats); + + expected_rebalance_cnt = 0; + /* variation ERROR_FIRST_HB: Second HB receives the assignment */ + if (variation == TEST_VARIATION_ERROR_SECOND_HB) { + expected_rebalance_cnt++; + rebalance_exp_event = RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS; + + /* variation ERROR_SECOND_HB: first HB assigned the partitions + * and second one acknowledges them and receives the + * fencing error. */ + rkmessage = rd_kafka_consumer_poll(c, 100); + TEST_ASSERT(!rkmessage, "No message should be returned"); + TEST_ASSERT( + rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected assign callback to be processed, but it wasn't"); + + TEST_ASSERT(rebalance_cnt == expected_rebalance_cnt, + "Expected %d rebalance events after assign " + "callback, got %d", + expected_rebalance_cnt, rebalance_cnt); + /* Ack is sent immediately after assignment completes. */ + expected_heartbeats++; + + TEST_SAY("Awaiting partition lost callback\n"); + /* Second HB acks receives the fenced error + * and loses partitions */ + expected_rebalance_cnt++; + rebalance_exp_event = RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS; + rebalance_exp_lost = rd_true; + + rkmessage = rd_kafka_consumer_poll(c, 100); + TEST_ASSERT(!rkmessage, "No message should be returned"); + TEST_ASSERT( + rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected revoke callback to be processed, but it wasn't"); + + TEST_ASSERT( + rebalance_cnt == expected_rebalance_cnt, + "Expected %d rebalance events after lost callback, got %d", + expected_rebalance_cnt, rebalance_cnt); + + /* Third HB assigns the partitions again */ + expected_heartbeats++; + } + + expected_rebalance_cnt++; + rebalance_exp_event = RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS; + rebalance_exp_lost = rd_false; + + TEST_SAY("Awaiting rebalance callback\n"); + /* Consume from c, partitions are lost if assigned */ + rkmessage = rd_kafka_consumer_poll(c, 500); + TEST_ASSERT(!rkmessage, "No message should be returned"); + TEST_ASSERT(rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected assign callback to be processed, but it wasn't"); + + TEST_ASSERT(rebalance_cnt == expected_rebalance_cnt, + "Expected %d total rebalance events, got %d", + expected_rebalance_cnt, rebalance_cnt); + + /* Ack for last assignment HB */ + expected_heartbeats++; + + TEST_SAY("Awaiting acknowledge heartbeat\n"); + TEST_ASSERT((found_heartbeats = + wait_all_heartbeats_done(mcluster, expected_heartbeats, + 100)) == expected_heartbeats, + "Expected %d heartbeats, got %d", expected_heartbeats, + found_heartbeats); + + expected_rebalance_cnt++; + rebalance_exp_event = RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS; + + /* Leave group HB */ + expected_heartbeats++; + /* Close c, no error is returned */ + TEST_CALL_ERR__(rd_kafka_consumer_close(c)); + TEST_ASSERT(rebalance_exp_event == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected revoke callback to be processed, but it wasn't"); + TEST_ASSERT(rebalance_cnt == expected_rebalance_cnt, + "Expected %d rebalance events, got %d", + expected_rebalance_cnt, rebalance_cnt); + + rd_kafka_destroy(c); + + TEST_SAY("Verifying leave group heartbeat\n"); + /* After closing the consumer, 1 heartbeat should been sent */ + TEST_ASSERT((found_heartbeats = + wait_all_heartbeats_done(mcluster, expected_heartbeats, + 0)) == expected_heartbeats, + "Expected %d heartbeats, got %d", expected_heartbeats, + found_heartbeats); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + TIMING_ASSERT(&timing, 100, 1500); + SUB_TEST_PASS(); +} + +/** + * @brief Test all kind of consumer fenced errors in a ConsumerGroupHeartbeat + * call. + * @sa test_variation_t + */ +static void do_test_consumer_group_heartbeat_fenced_errors(void) { + rd_kafka_resp_err_t fenced_errors[] = { + RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID, + RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH}; + size_t i; + test_variation_t j; + for (i = 0; i < RD_ARRAY_SIZE(fenced_errors); i++) { + for (j = TEST_VARIATION_ERROR_FIRST_HB; j < TEST_VARIATION__CNT; + j++) + do_test_consumer_group_heartbeat_fenced_error( + fenced_errors[i], j); + } +} + +/** + * @enum test_variation_unknown_topic_id_t + * @brief Variations for `do_test_metadata_unknown_topic_id_tests`. + */ +typedef enum test_variation_unknown_topic_id_t { + /* One topic, UNKNOWN_TOPIC_ID is given until it's not. */ + TEST_VARIATION_UNKNOWN_TOPIC_ID_ONE_TOPIC = 0, + /* Two topics, first has UNKNOWN_TOPIC_ID error, second one exists. */ + TEST_VARIATION_UNKNOWN_TOPIC_ID_TWO_TOPICS = 1, + TEST_VARIATION_UNKNOWN_TOPIC_ID__CNT, +} test_variation_unknown_topic_id_t; + +static const char * +test_variation_unknown_topic_id_name(test_variation_t variation) { + switch (variation) { + case TEST_VARIATION_UNKNOWN_TOPIC_ID_ONE_TOPIC: + return "one topic"; + case TEST_VARIATION_UNKNOWN_TOPIC_ID_TWO_TOPICS: + return "two topics"; + default: + rd_assert(!*"Unknown test variation (unknown topic id)"); + return NULL; + } +} + +/** + * @brief Test consumer group behavior with missing topic id when retrieving + * metadata for assigned topics. + * ensuring: + * - initially a partial acknoledgement is started, with an empty list + * (variation 0) or a single topic (variation 1) + * - fetch doesn't start until broker returns an unknown topic id error + * - when error isn't returned anymore the client finishes assigning + * the partition and reads a message. + * + * @param variation Test variation, see `test_variation_unknown_topic_id_t`. + */ +static void do_test_metadata_unknown_topic_id_error( + test_variation_unknown_topic_id_t variation) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_t *c; + test_timing_t timing; + const char *topic = "do_test_metadata_unknown_topic_id_error"; + const char *topic2 = "do_test_metadata_unknown_topic_id_error2"; + rd_kafka_topic_partition_list_t *expected_assignment; + + SUB_TEST_QUICK("variation: %s", + test_variation_unknown_topic_id_name(variation)); + + expected_assignment = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(expected_assignment, topic, 0); + if (variation == TEST_VARIATION_UNKNOWN_TOPIC_ID_TWO_TOPICS) { + rd_kafka_topic_partition_list_add(expected_assignment, topic2, + 0); + } + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 500); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + if (variation == TEST_VARIATION_UNKNOWN_TOPIC_ID_TWO_TOPICS) { + rd_kafka_mock_topic_create(mcluster, topic2, 1, 1); + } + + c = create_consumer(bootstraps, topic, rd_false); + + /* Seed the topic with messages */ + test_produce_msgs_easy_v(topic, 0, 0, 0, 1, 1000, "bootstrap.servers", + bootstraps, NULL); + + TIMING_START(&timing, "do_test_metadata_unknown_topic_id_error"); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + if (variation == TEST_VARIATION_UNKNOWN_TOPIC_ID_TWO_TOPICS) { + rd_kafka_topic_partition_list_add(subscription, topic2, + RD_KAFKA_PARTITION_UA); + } + + rd_kafka_mock_topic_set_error(mcluster, topic, + RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID); + + TEST_SAY("Subscribing to topic\n"); + TEST_CALL_ERR__(rd_kafka_subscribe(c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + TEST_SAY( + "Cannot fetch until Metadata calls replies with " + "UNKNOWN_TOPIC_ID\n"); + test_consumer_poll_no_msgs("no messages", c, 0, 1000); + + rd_kafka_mock_topic_set_error(mcluster, topic, + RD_KAFKA_RESP_ERR_NO_ERROR); + + TEST_SAY("Reconciliation and fetch is now possible\n"); + test_consumer_poll_timeout("message", c, 0, 0, 0, 1, NULL, 2000); + + TEST_CALL_ERR__(rd_kafka_assignment(c, &assignment)); + TEST_ASSERT(assignment != NULL); + TEST_ASSERT(!test_partition_list_cmp(assignment, expected_assignment), + "Expected assignment not seen, got %d partitions", + assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + rd_kafka_topic_partition_list_destroy(expected_assignment); + + rd_kafka_destroy(c); + test_mock_cluster_destroy(mcluster); + + TIMING_ASSERT(&timing, 500, 4000); + SUB_TEST_PASS(); +} + +/** + * @brief Test these variations of a UNKNOWN_TOPIC_ID in a Metadata call + * before reconciliation. + * @sa test_variation_unknown_topic_id_t + */ +static void do_test_metadata_unknown_topic_id_tests(void) { + test_variation_unknown_topic_id_t i; + for (i = TEST_VARIATION_UNKNOWN_TOPIC_ID_ONE_TOPIC; + i < TEST_VARIATION_UNKNOWN_TOPIC_ID__CNT; i++) { + do_test_metadata_unknown_topic_id_error(i); + } +} + +static void do_test_adherence_to_hb_interval(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_t *c; + const char *topic = + test_mk_topic_name("do_test_adherence_to_hb_interval", 1); + rd_kafka_conf_t *conf; + size_t heartbeat_request_count = 0; + + SUB_TEST_QUICK("do_test_adherence_to_hb_interval"); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 1000); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "auto.commit.interval.ms", "100"); + c = test_create_consumer(topic, NULL, conf, NULL); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_SAY("Subscribing to topic\n"); + TEST_CALL_ERR__(rd_kafka_subscribe(c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + TEST_SAY("Subscription done, waiting for heartbeats\n"); + + rd_sleep(2); /* Sleep to ensure that some HB are sent */ + + heartbeat_request_count = test_mock_get_matching_request_cnt( + mcluster, is_heartbeat_request, NULL); + TEST_SAY("Heartbeat request count: %zu\n", heartbeat_request_count); + + /* Assert that we received the expected number of heartbeats */ + TEST_ASSERT(heartbeat_request_count >= 3 && + heartbeat_request_count <= 5, + "Expected between 3 and 5 heartbeats, got %zu", + heartbeat_request_count); + + rd_kafka_mock_stop_request_tracking(mcluster); + + rd_kafka_destroy(c); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +typedef enum do_test_quick_unsubscribe_variation_t { + /* No mock cluster, no coordinator available. */ + DO_TEST_QUICK_UNSUBSCRIBE_VARIATION_NO_CLUSTER = 0, + /* Mock cluster is ready */ + DO_TEST_QUICK_UNSUBSCRIBE_VARIATION_CLUSTER_READY = 1, + DO_TEST_QUICK_UNSUBSCRIBE_VARIATION__CNT +} do_test_quick_unsubscribe_variation_t; + +/** + * @brief A series of subscribe and unsubscribe call shouldn't cause + * assert failures. + * + * @param variation Test variation. + * + * @sa `do_test_quick_unsubscribe_variation_t` + */ +static void +do_test_quick_unsubscribe(do_test_quick_unsubscribe_variation_t variation) { + int i; + rd_kafka_t *c; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_mock_cluster_t *mcluster = NULL; + const char *bootstraps = "localhost:9999"; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + + SUB_TEST_QUICK( + "%s", variation == DO_TEST_QUICK_UNSUBSCRIBE_VARIATION_NO_CLUSTER + ? "no cluster" + : "mock cluster ready"); + + if (variation == DO_TEST_QUICK_UNSUBSCRIBE_VARIATION_NO_CLUSTER) { + test_curr->is_fatal_cb = error_is_fatal_cb; + allowed_error = RD_KAFKA_RESP_ERR__TRANSPORT; + } else if (variation == + DO_TEST_QUICK_UNSUBSCRIBE_VARIATION_CLUSTER_READY) { + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + } + + c = create_consumer(bootstraps, topic, rd_true); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + for (i = 0; i < 2; i++) { + TEST_CALL_ERR__(rd_kafka_subscribe(c, subscription)); + TEST_CALL_ERR__(rd_kafka_unsubscribe(c)); + } + + rd_kafka_topic_partition_list_destroy(subscription); + rd_kafka_destroy(c); + RD_IF_FREE(mcluster, test_mock_cluster_destroy); + + test_curr->is_fatal_cb = NULL; + allowed_error = RD_KAFKA_RESP_ERR_NO_ERROR; + SUB_TEST_PASS(); +} + +/** + * @brief Test all `do_test_quick_unsubscribe` variations. + * + * @sa `do_test_quick_unsubscribe_variation_t` + */ +static void do_test_quick_unsubscribe_tests(void) { + do_test_quick_unsubscribe_variation_t variation; + for (variation = DO_TEST_QUICK_UNSUBSCRIBE_VARIATION_NO_CLUSTER; + variation < DO_TEST_QUICK_UNSUBSCRIBE_VARIATION__CNT; + variation++) { + do_test_quick_unsubscribe(variation); + } +} + +static int max_poll_inject_done = 0; + +/** + * @brief Log interception used to deterministically force the max-poll rejoin + * race from the test, without modifying library code. + * + * The "Expediting next heartbeat ... max poll interval exceeded" line is + * emitted on the main thread when the poll interval is exceeded and + * F_WAIT_REJOIN is set. This sleep reproduces the *pre-fix* race: back then the + * leave-group heartbeat was sent immediately at that point, and sleeping here + * (synchronously, on the main thread) let the broker thread deliver the leave + * reply, which was then processed on the next serve -- running consumer_reset + * and clearing F_WAIT_REJOIN BEFORE consumer_serve acted on it -- stranding the + * member steady at epoch 0 so its next heartbeat was a malformed (re-)join. + * The fix defers the leave until the assignment is revoked, so the consumer + * now rejoins cleanly even with this injected delay. + */ +static void max_poll_rejoin_delay_log_cb(const rd_kafka_t *rk, + int level, + const char *fac, + const char *buf) { + if (!max_poll_inject_done && strstr(buf, "Expediting next heartbeat") && + strstr(buf, "max poll interval exceeded")) { + max_poll_inject_done = 1; + fprintf(stderr, + "### TEST: injecting 500ms delay after leave to force " + "the rejoin race\n"); + rd_usleep(500 * 1000, NULL); + } +} + +/** + * @brief After the poll timer (max.poll.interval.ms) expires the consumer + * leaves the group and must rejoin once polling resumes. Verify the + * rejoin is performed as a proper (re-)join, the consumer does not end + * up in a fatal state, and it re-acquires its assignment and consumes + * all the records. + * + * The topic is kept empty while the rejoin race is forced (so active fetching + * does not perturb the timing); the records are produced only afterwards. A + * regression strands the member at epoch 0 and the next heartbeat is rejected + * with a fatal INVALID_REQUEST -- which is detected first in the poll loop and + * fails the test fast. With the fix the consumer rejoins and (auto-commit + * disabled -> earliest) consumes the full set of records. + */ +static void do_test_max_poll_interval_rejoin_consume(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_t *c; + rd_kafka_conf_t *conf; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + uint64_t testid = test_id_generate(); + const int msgcnt = 100; + rd_kafka_resp_err_t fatal_err; + char errstr[512]; + int64_t deadline; + rd_kafka_message_t *rkm; + rd_kafka_topic_partition_list_t *assignment = NULL; + rd_bool_t assigned = rd_false; + int received = 0; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 500); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + + max_poll_inject_done = 0; + test_conf_init(&conf, NULL, 30); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "auto.offset.reset", "earliest"); + /* Disable auto-commit so the re-assigned partition has no committed + * offset after the rejoin and deterministically resets to earliest. */ + test_conf_set(conf, "enable.auto.commit", "false"); + test_conf_set(conf, "max.poll.interval.ms", "3000"); + /* Intercept logs to force the rejoin race, see + * max_poll_rejoin_delay_log_cb(). The interceptor enables the "cgrp" + * debug context on top of any set via the TEST_DEBUG env variable. */ + const char *debug_contexts[2] = {"cgrp", NULL}; + test_conf_set_log_interceptor(conf, max_poll_rejoin_delay_log_cb, + debug_contexts); + c = test_create_consumer(topic, NULL, conf, NULL); + + test_consumer_subscribe(c, topic); + + /* Reach steady membership on an empty topic: the partition is assigned + * but there is nothing to fetch, so the forced rejoin race below is + * not perturbed by fetch traffic. */ + TEST_SAY("Waiting for steady assignment\n"); + deadline = test_clock() + 10 * 1000000; + while (test_clock() < deadline) { + rkm = rd_kafka_consumer_poll(c, 200); + if (rkm) + rd_kafka_message_destroy(rkm); + if (!rd_kafka_assignment(c, &assignment) && assignment && + assignment->cnt > 0) { + rd_kafka_topic_partition_list_destroy(assignment); + assignment = NULL; + assigned = rd_true; + break; + } + RD_IF_FREE(assignment, rd_kafka_topic_partition_list_destroy); + assignment = NULL; + } + TEST_ASSERT(assigned, + "Timed out waiting for steady assignment before the test " + "could proceed"); + + TEST_SAY("Stalling for 4s (> max.poll.interval.ms=3s)\n"); + rd_sleep(4); + + /* Produce only now, after the race window, so the records are consumed + * by the (re-)joined member rather than prefetched during the race. */ + test_produce_msgs_easy_v(topic, testid, RD_KAFKA_PARTITION_UA, 0, + msgcnt, 64, "bootstrap.servers", bootstraps, + NULL); + + /* Resume: a regression goes fatal here (detected first, fails fast); + * with the fix the consumer rejoins and consumes all records. */ + TEST_SAY( + "Resuming poll; consumer must rejoin and consume all %d " + "records\n", + msgcnt); + deadline = test_clock() + 20 * 1000000; + while (received < msgcnt && test_clock() < deadline) { + fatal_err = rd_kafka_fatal_error(c, errstr, sizeof(errstr)); + if (fatal_err) + TEST_FAIL( + "Consumer went fatal after max.poll.interval.ms " + "expiry instead of rejoining: %s: %s", + rd_kafka_err2name(fatal_err), errstr); + rkm = rd_kafka_consumer_poll(c, 200); + if (!rkm) + continue; + if (!rkm->err) + received++; + rd_kafka_message_destroy(rkm); + } + + TEST_ASSERT(received == msgcnt, + "Expected to consume all %d records after rejoining, " + "got %d: consumer failed to fully recover", + msgcnt, received); + + test_consumer_close(c); + rd_kafka_destroy(c); + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +/** + * @brief A GROUP_ID_NOT_FOUND received on a regular heartbeat that is still in + * flight when the leave heartbeat has already been sent must be ignored + * (the member is on its way out of the group), not turned into a fatal + * error. + * + * The window is forced deterministically: the next regular heartbeat is + * answered with GROUP_ID_NOT_FOUND, but its response is held back with a large + * RTT. While it is in flight the consumer unsubscribes, which sends the leave + * heartbeat and enters the leaving state. The mock sends a connection's + * responses in order (head-of-line), so the delayed GROUP_ID_NOT_FOUND is + * processed before the leave response leaves the leaving state -- exercising + * the skip path. A regression treats it as fatal instead. + */ +static void do_test_group_id_not_found_while_leaving(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_t *c; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + rd_kafka_topic_partition_list_t *assignment = NULL; + rd_bool_t assigned = rd_false; + rd_kafka_resp_err_t fatal_err; + char errstr[512]; + int64_t deadline; + rd_kafka_message_t *rkm; + int found_heartbeats; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 500); + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + + c = create_consumer(bootstraps, topic, rd_false); + test_consumer_subscribe(c, topic); + + TEST_SAY("Waiting for steady assignment\n"); + deadline = test_clock() + 10 * 1000000; + while (test_clock() < deadline) { + rkm = rd_kafka_consumer_poll(c, 200); + if (rkm) + rd_kafka_message_destroy(rkm); + if (!rd_kafka_assignment(c, &assignment) && assignment && + assignment->cnt > 0) { + rd_kafka_topic_partition_list_destroy(assignment); + assignment = NULL; + assigned = rd_true; + break; + } + RD_IF_FREE(assignment, rd_kafka_topic_partition_list_destroy); + assignment = NULL; + } + TEST_ASSERT(assigned, + "Timed out waiting for steady assignment before the test " + "could proceed"); + TEST_SAY("Steady assignment reached\n"); + + /* Answer the next regular heartbeat with GROUP_ID_NOT_FOUND, holding + * its response back long enough to land after the leave heartbeat is + * sent (which happens within milliseconds of unsubscribe) but well + * below the in-flight request timeout so the response is not discarded + * as a client-side timeout. While that heartbeat is in flight no + * further regular heartbeat is sent, so it is the one that will carry + * the error. */ + rd_kafka_mock_start_request_tracking(mcluster); + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND, 500); + + TEST_SAY("Driving the poisoned heartbeat in flight\n"); + found_heartbeats = wait_all_heartbeats_done(mcluster, 1, 100); + TEST_ASSERT(found_heartbeats == 1, "Expected 1 heartbeat, got %d", + found_heartbeats); + + TEST_SAY("Unsubscribing while the heartbeat is in flight\n"); + TEST_CALL_ERR__(rd_kafka_unsubscribe(c)); + + /* Poll past the RTT so the delayed GROUP_ID_NOT_FOUND is processed + * (must be skipped) followed by the leave response. */ + deadline = test_clock() + 4 * 1000000; + fatal_err = RD_KAFKA_RESP_ERR_NO_ERROR; + while (test_clock() < deadline) { + rkm = rd_kafka_consumer_poll(c, 200); + if (rkm) + rd_kafka_message_destroy(rkm); + fatal_err = rd_kafka_fatal_error(c, errstr, sizeof(errstr)); + if (fatal_err) + break; + } + + TEST_ASSERT(!fatal_err, + "GROUP_ID_NOT_FOUND while leaving must be ignored, but the " + "consumer went fatal: %s: %s", + rd_kafka_err2name(fatal_err), errstr); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_consumer_close(c); + rd_kafka_destroy(c); + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +int main_0147_consumer_group_consumer_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + if (test_consumer_group_protocol_classic()) { + TEST_SKIP("Test only for group.protocol=consumer\n"); + return 0; + } + + do_test_max_poll_interval_rejoin_consume(); + + do_test_consumer_group_heartbeat_fatal_errors(); + + do_test_group_id_not_found_while_leaving(); + + do_test_consumer_group_heartbeat_retriable_errors(); + + do_test_consumer_group_heartbeat_fenced_errors(); + + do_test_metadata_unknown_topic_id_tests(); + + do_test_adherence_to_hb_interval(); + + do_test_quick_unsubscribe_tests(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0148-offset_fetch_commit_error_mock.c b/lib/librdkafka-2.15.0/tests/0148-offset_fetch_commit_error_mock.c new file mode 100644 index 00000000000..9cb0fb72acd --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0148-offset_fetch_commit_error_mock.c @@ -0,0 +1,569 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +#include + + +static mtx_t log_lock; +static cnd_t log_cnd; +static rd_bool_t revocation_done = rd_false; +static int got_stale_member_epoch_error = 0; +static rd_bool_t trigger_consumer_close = rd_false; +static int ack_target_assignment_count = 0; + +static void +log_cb(const rd_kafka_t *rk, int level, const char *fac, const char *buf) { + if (strstr(buf, + "assignment operations done in join-state " + "wait-incr-unassign-to-complete")) { + mtx_lock(&log_lock); + revocation_done = rd_true; + cnd_signal(&log_cnd); + mtx_unlock(&log_lock); + } +} + +/** + * @brief A stale member error during an OffsetFetch should cause + * to retry the operation just after next ConsumerGroupHeartbeat + * response. + * The offset fetch eventually succeeds and the consumer can + * start from the committed offset. + */ +void do_test_OffsetFetch_stale_member_epoch_error( + rd_kafka_mock_cluster_t *mcluster, + const char *bootstraps) { + const char *topic = test_mk_topic_name(__FUNCTION__, 1); + rd_kafka_t *producer, *first_consumer, *second_consumer; + rd_kafka_conf_t *conf, *producer_conf; + uint64_t testid = test_id_generate(); + const int msgcnt = 5; + test_msgver_t mv; + + SUB_TEST_QUICK(); + + test_conf_init(&conf, NULL, 30); + test_conf_set(conf, "bootstrap.servers", bootstraps); + + /* Producer */ + producer_conf = rd_kafka_conf_dup(conf); + rd_kafka_conf_set_dr_msg_cb(producer_conf, test_dr_msg_cb); + producer = test_create_handle(RD_KAFKA_PRODUCER, producer_conf); + rd_kafka_mock_topic_create(mcluster, topic, 1, 2); + test_produce_msgs2(producer, topic, testid, 0, 0, msgcnt, NULL, 0); + rd_kafka_flush(producer, -1); + + /* Consumer */ + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "group.protocol", "consumer"); + first_consumer = + test_create_consumer(topic, NULL, rd_kafka_conf_dup(conf), NULL); + test_consumer_subscribe(first_consumer, topic); + test_consumer_poll("before consume error", first_consumer, testid, -1, + 0, msgcnt, NULL); + test_consumer_close(first_consumer); + + /* Produce again */ + test_produce_msgs2(producer, topic, testid, 0, msgcnt, msgcnt, NULL, 0); + rd_kafka_flush(producer, -1); + + /* Set OffsetFetch errors */ + rd_kafka_mock_push_request_errors(mcluster, RD_KAFKAP_OffsetFetch, 5, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH); + + /* Consume again*/ + test_msgver_init(&mv, testid); + second_consumer = test_create_consumer(topic, NULL, conf, NULL); + test_consumer_subscribe(second_consumer, topic); + test_consumer_poll("receive second batch", second_consumer, testid, -1, + msgcnt, msgcnt, &mv); + test_msgver_verify("verify second batch", &mv, TEST_MSGVER_ALL, msgcnt, + msgcnt); + test_msgver_clear(&mv); + test_consumer_close(second_consumer); + + /* Destroy */ + rd_kafka_destroy(first_consumer); + rd_kafka_destroy(second_consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + +typedef enum do_test_OffsetCommit_manual_error_variation_s { + /** commit stored offsets */ + TEST_MANUAL_COMMIT_ERROR_VARIATION_STORE_OFFSET_AUTOMATICALLY = 0, + /** commit passed offsets */ + TEST_MANUAL_COMMIT_ERROR_VARIATION_STORE_OFFSET_MANUALLY = 1, + TEST_MANUAL_COMMIT_ERROR_VARIATION__CNT, +} do_test_OffsetCommit_manual_error_variation_t; + +/** + * @brief Doing a manual commits that returns error \p expected_err + * should return the error to the caller, even if the error + * is a partition level error. + * These errors aren't retried. + */ +void do_test_OffsetCommit_manual_error( + rd_kafka_mock_cluster_t *mcluster, + const char *bootstraps, + rd_kafka_resp_err_t expected_err, + do_test_OffsetCommit_manual_error_variation_t variation) { + rd_kafka_t *consumer; + test_msgver_t mv; + rd_kafka_conf_t *conf; + const char *topic = test_mk_topic_name(__FUNCTION__, 1); + uint64_t testid = test_id_generate(); + const int msgcnt = 5; + rd_kafka_resp_err_t err; + rd_kafka_topic_partition_list_t *to_commit = NULL; + + SUB_TEST_QUICK(); + + rd_kafka_mock_topic_create(mcluster, topic, 1, 1); + + test_conf_init(&conf, NULL, 30); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "enable.auto.commit", "false"); + test_conf_set(conf, "group.protocol", "consumer"); + + /* Seed the topic with messages */ + test_produce_msgs_easy_v(topic, testid, 0, 0, msgcnt, 0, + "bootstrap.servers", bootstraps, NULL); + + /* Consume same messages */ + consumer = test_create_consumer(topic, NULL, conf, NULL); + test_consumer_subscribe(consumer, topic); + test_msgver_init(&mv, testid); + test_consumer_poll("receive first batch", consumer, testid, -1, 0, + msgcnt, &mv); + test_msgver_verify("verify first batch", &mv, TEST_MSGVER_ALL, 0, + msgcnt); + test_msgver_clear(&mv); + + /* Set OffsetCommit errors */ + rd_kafka_mock_push_request_errors(mcluster, RD_KAFKAP_OffsetCommit, 1, + expected_err); + + if (variation == + TEST_MANUAL_COMMIT_ERROR_VARIATION_STORE_OFFSET_MANUALLY) { + /* Variation 1: pass offsets to commit */ + to_commit = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(to_commit, topic, 0)->offset = + msgcnt; + } + + /* Sync commit */ + err = rd_kafka_commit(consumer, to_commit, rd_false); + TEST_ASSERT(err == expected_err, "Expected error %s, got %s", + rd_kafka_err2name(expected_err), rd_kafka_err2name(err)); + + /* Retry it, this time it should work */ + err = rd_kafka_commit(consumer, to_commit, rd_false); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected error %s, got %s", + rd_kafka_err2name(RD_KAFKA_RESP_ERR_NO_ERROR), + rd_kafka_err2name(err)); + + RD_IF_FREE(to_commit, rd_kafka_topic_partition_list_destroy); + + rd_kafka_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * Scenarios: + * during_revocation: the auto-commit is triggered by a revocation, + * otherwise it's triggered by the consumer close. + * session_times_out: Session times out giving UNKNOWN_MEMBER_ID, otherwise + * commit succeeds after last STALE_MEMBER_EPOCH. When + * session times out the auto-commit fails and messages + * are consumed again. + */ +typedef enum do_test_OffsetCommit_automatic_stale_member_epoch_error_variation_t { + /** during_revocation=false, session_times_out=false */ + TEST_AUTO_COMMIT_STALE_MEMBER_EPOCH_VARIATION_NO_REVOKE_NO_TIMEOUT = 0, + /** during_revocation=false, session_times_out=true */ + TEST_AUTO_COMMIT_STALE_MEMBER_EPOCH_VARIATION_NO_REVOKE_WITH_TIMEOUT = + 1, + /** during_revocation=true, session_times_out=false */ + TEST_AUTO_COMMIT_STALE_MEMBER_EPOCH_VARIATION_REVOKE_NO_TIMEOUT = 2, + /** during_revocation=true, session_times_out=true */ + TEST_AUTO_COMMIT_STALE_MEMBER_EPOCH_VARIATION_REVOKE_WITH_TIMEOUT = 3, + TEST_AUTO_COMMIT_STALE_MEMBER_EPOCH_VARIATION__CNT, +} do_test_OffsetCommit_automatic_stale_member_epoch_error_variation_t; + +/** + * @brief When a partition is revoked, with auto-commit enabled, + * if the RPC returns STALE_MEMBER_EPOCH for one of the + * partitions, it should be retried until the member is + * fenced. + */ +void do_test_OffsetCommit_automatic_stale_member_epoch_error( + rd_kafka_mock_cluster_t *mcluster, + const char *bootstraps, + do_test_OffsetCommit_automatic_stale_member_epoch_error_variation_t + variation) { + rd_kafka_t *consumer; + test_msgver_t mv; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *target_assignment_partitions; + const char *topic = test_mk_topic_name(__FUNCTION__, 1); + uint64_t testid = test_id_generate(); + const int msgcnt = 5; + const int session_timeout_ms = 3000; + const int heartbeat_interval_ms = 1000; + rd_bool_t during_revocation = (variation / 2) == 1; + rd_bool_t session_times_out = (variation % 2) == 1; + const char *debug_contexts[2] = {"cgrp", NULL}; + + SUB_TEST_QUICK("during_revocation=%s, session_times_out=%s", + RD_STR_ToF(during_revocation), + RD_STR_ToF(session_times_out)); + + mtx_init(&log_lock, mtx_plain); + cnd_init(&log_cnd); + + rd_kafka_mock_topic_create(mcluster, topic, 2, 2); + rd_kafka_mock_coordinator_set(mcluster, "group", topic, 1); + rd_kafka_mock_set_group_consumer_session_timeout_ms(mcluster, + session_timeout_ms); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms( + mcluster, heartbeat_interval_ms); + + test_conf_init(&conf, NULL, 30); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "enable.auto.commit", "true"); + test_conf_set(conf, "auto.commit.interval.ms", "1000"); + test_conf_set(conf, "group.protocol", "consumer"); + test_conf_set_log_interceptor(conf, log_cb, debug_contexts); + + /* Seed the topic with messages */ + test_produce_msgs_easy_v(topic, testid, 0, 0, 3, 0, "bootstrap.servers", + bootstraps, NULL); + test_produce_msgs_easy_v(topic, testid, 1, 0, 2, 0, "bootstrap.servers", + bootstraps, NULL); + + /* Consume same messages */ + consumer = + test_create_consumer(topic, NULL, rd_kafka_conf_dup(conf), NULL); + test_consumer_subscribe(consumer, topic); + test_msgver_init(&mv, testid); + test_consumer_poll_exact("receive first batch", consumer, testid, -1, 0, + msgcnt, rd_true, &mv); + test_msgver_verify("verify first batch", &mv, TEST_MSGVER_PER_PART, 0, + msgcnt); + test_msgver_clear(&mv); + + + rd_kafka_mock_clear_request_errors(mcluster, RD_KAFKAP_OffsetCommit); + + /* First sequence of stale member epoch for 4 s */ + rd_kafka_mock_push_request_errors(mcluster, RD_KAFKAP_OffsetCommit, 4, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH); + + if (during_revocation) { + /* Changing target assignment to partition 0 only, + * partition revoked and automatically committed, + * but the commit fails. */ + target_assignment_partitions = + rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(target_assignment_partitions, + topic, 0); + test_mock_cluster_member_assignment( + mcluster, 1, consumer, target_assignment_partitions); + rd_kafka_topic_partition_list_destroy( + target_assignment_partitions); + + mtx_lock(&log_lock); + while (!revocation_done) + cnd_timedwait_ms(&log_cnd, &log_lock, 500); + revocation_done = rd_false; + mtx_unlock(&log_lock); + } + + if (session_times_out) { + /* Simulate a session timeout after that */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_OffsetCommit, 1, + RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID, + session_timeout_ms + heartbeat_interval_ms); + } + + /* Otherwise partition is committed before leaving the group */ + test_consumer_close(consumer); + rd_kafka_destroy(consumer); + + + /* Reset mock assignor to automatic */ + rd_kafka_mock_cgrp_consumer_target_assignment(mcluster, topic, NULL); + + consumer = test_create_consumer(topic, NULL, conf, NULL); + test_consumer_subscribe(consumer, topic); + + if (session_times_out) { + /* Messages are consumed again because the commit failed */ + test_msgver_init(&mv, testid); + test_consumer_poll_exact("messages consumed again", consumer, + testid, -1, 0, msgcnt, rd_true, &mv); + test_msgver_verify("messages consumed again", &mv, + TEST_MSGVER_PER_PART, 0, msgcnt); + test_msgver_clear(&mv); + } else { + /* No message should be consumed after the autocommit */ + test_consumer_poll_no_msgs("no messages", consumer, testid, + 200); + } + test_consumer_close(consumer); + rd_kafka_destroy(consumer); + rd_kafka_mock_clear_request_errors(mcluster, RD_KAFKAP_OffsetCommit); + + mtx_destroy(&log_lock); + cnd_destroy(&log_cnd); + + SUB_TEST_PASS(); +} + +static void log_cb_closing_issue(const rd_kafka_t *rk, + int level, + const char *fac, + const char *buf) { + if (strstr(buf, "Acknowledging target assignment")) { + mtx_lock(&log_lock); + ack_target_assignment_count++; + if (ack_target_assignment_count == 2) { + trigger_consumer_close = rd_true; + cnd_signal(&log_cnd); + } + mtx_unlock(&log_lock); + } + if (strstr(buf, "unable to OffsetCommit") && + strstr(buf, "Broker: The member epoch is stale")) { + mtx_lock(&log_lock); + got_stale_member_epoch_error++; + mtx_unlock(&log_lock); + } +} + +/** + * @brief This test checks that when a consumer acknowledges revocations and + * sends a heartbeat just before leaving the group, the heartbeat response may + * still be in flight while the leave process begins. In this scenario, the + * heartbeat response must update the member epoch rather than being discarded. + * Otherwise, subsequent commit requests (which will be required for leaving) + * may fail with a stale member epoch error. + * + * Sequence of events: + * 1. Consumer is subscribed to topic1 and topic2 and has received messages + * 2. Consumer changes subscription to only topic1, this triggers revocation of + * topic2 partitions. + * 3. Consumer acknowledges the revocation and sends a heartbeat. The heartbeat + * response is delayed (simulated by mock). Due to this, the member epoch on + * the broker end is increased but the consumer has not received it yet. + * 4. Meanwhile, the consumer receives messages from topic1. + * 5. The consumer is closed, this triggers a leave group which includes an + * offset commit request for the received messages. + * 6. This triggers a stale member epoch error since the consumer is not aware + * of the increased member epoch on the broker end. + * 7. The stale member epoch error triggers ConsumerGroupHeartbeat request + * which receives to receive the latest member epoch. + * 8. The offset commit is retried and succeeds. + */ +void do_test_consumer_inflight_heartbeat_on_leave( + rd_kafka_mock_cluster_t *mcluster, + const char *bootstraps) { + + char topic1[256], topic2[256]; + rd_kafka_t *producer, *consumer; + rd_kafka_conf_t *conf, *producer_conf; + uint64_t testid = test_id_generate(); + const int msgcnt = 5; + test_msgver_t mv; + const char *debug_contexts[3] = {"cgrp", NULL}; + int64_t close_start, close_end; + const int session_timeout_ms = 3000; + const int heartbeat_rtt_ms = 200; + + SUB_TEST_QUICK(); + + mtx_init(&log_lock, mtx_plain); + cnd_init(&log_cnd); + + strcpy(topic1, test_mk_topic_name("topic1", 1)); + strcpy(topic2, test_mk_topic_name("topic2", 1)); + + test_conf_init(&conf, NULL, 30); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.protocol", "consumer"); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "enable.auto.commit", "true"); + test_conf_set(conf, "auto.commit.interval.ms", "2000"); + test_conf_set(conf, "fetch.wait.max.ms", "100"); + test_conf_set_log_interceptor(conf, log_cb_closing_issue, + debug_contexts); + + rd_kafka_mock_coordinator_set(mcluster, "group", topic1, 1); + rd_kafka_mock_set_group_consumer_session_timeout_ms(mcluster, + session_timeout_ms); + rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 500); + + /* Producer Initialization */ + producer_conf = rd_kafka_conf_dup(conf); + rd_kafka_conf_set_dr_msg_cb(producer_conf, test_dr_msg_cb); + producer = test_create_handle(RD_KAFKA_PRODUCER, producer_conf); + + /* Create topic1 and produce few messages */ + rd_kafka_mock_topic_create(mcluster, topic1, 1, 1); + test_produce_msgs2(producer, topic1, testid, 0, 0, msgcnt, NULL, 0); + rd_kafka_flush(producer, -1); + + /* Create topic2 and produce few messages */ + rd_kafka_mock_topic_create(mcluster, topic2, 1, 1); + test_produce_msgs2(producer, topic2, testid, 0, 0, msgcnt, NULL, 0); + rd_kafka_flush(producer, -1); + + /* Consumer: subscribe to both topics */ + TEST_SAY("Group id: %s\n", topic1); + consumer = + test_create_consumer(topic1, NULL, rd_kafka_conf_dup(conf), NULL); + test_consumer_subscribe_multi(consumer, 2, topic1, topic2); + + /* Poll and verify messages produced to both topics */ + test_msgver_init(&mv, testid); + test_consumer_poll("read from both topics", consumer, testid, -1, 0, + 2 * msgcnt, &mv); + test_msgver_clear(&mv); + + /* Change subscription to only topic1 to trigger revocation */ + test_consumer_subscribe(consumer, topic1); + + /* Set ConsumerGroupHeartbeat RTT to heartbeat_rtt_ms */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ConsumerGroupHeartbeat, 2, + RD_KAFKA_RESP_ERR_NO_ERROR, heartbeat_rtt_ms, + RD_KAFKA_RESP_ERR_NO_ERROR, heartbeat_rtt_ms); + + /* Produce few more messages to topic1 so that we can trigger + auto-commit later which will give stale member epoch error */ + test_produce_msgs2(producer, topic1, testid, 0, 0, msgcnt, NULL, 0); + rd_kafka_flush(producer, -1); + + /* Wait for log callback to trigger consumer close after second + "Acknowledging target assignment" */ + mtx_lock(&log_lock); + while (!trigger_consumer_close) + cnd_timedwait_ms(&log_cnd, &log_lock, 500); + mtx_unlock(&log_lock); + + /* Poll and verify the produced messages */ + test_msgver_init(&mv, testid); + test_consumer_poll("read topic1", consumer, testid, -1, 0, msgcnt, &mv); + test_msgver_clear(&mv); + + /* Close consumer which will trigger leave group and auto-commit + The auto-commit will get stale member epoch error and will retry + after receiving the heartbeat response with the latest member + epoch */ + close_start = test_clock(); + test_consumer_close(consumer); + close_end = test_clock(); + + /* Verify that we got exactly one stale member epoch error */ + mtx_lock(&log_lock); + TEST_ASSERT(got_stale_member_epoch_error == 1, + "Expected 1 stale member epoch error, got %d", + got_stale_member_epoch_error); + mtx_unlock(&log_lock); + + /* Verify that the consumer closed within session timeout, if it reaches + session timeout which means that the member is kicked out of the + group. */ + TEST_ASSERT((close_end - close_start) < session_timeout_ms * 1000, + "Consumer did not close within 2s, took %" PRId64 " us", + (close_end - close_start)); + + rd_kafka_destroy(consumer); + rd_kafka_destroy(producer); + + mtx_destroy(&log_lock); + cnd_destroy(&log_cnd); + + SUB_TEST_PASS(); +} + +int main_0148_offset_fetch_commit_error_mock(int argc, char **argv) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + int i; + + TEST_SKIP_MOCK_CLUSTER(0); + + if (test_consumer_group_protocol_classic()) { + TEST_SKIP( + "Test not meaningful with 'classic' consumer group " + "protocol\n"); + return 0; + } + + mcluster = test_mock_cluster_new(3, &bootstraps); + + do_test_OffsetFetch_stale_member_epoch_error(mcluster, bootstraps); + + for (i = 0; i < TEST_MANUAL_COMMIT_ERROR_VARIATION__CNT; i++) { + do_test_OffsetCommit_manual_error( + mcluster, bootstraps, RD_KAFKA_RESP_ERR_STALE_MEMBER_EPOCH, + i); + do_test_OffsetCommit_manual_error( + mcluster, bootstraps, RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID, + i); + } + + for (i = 0; i < TEST_AUTO_COMMIT_STALE_MEMBER_EPOCH_VARIATION__CNT; i++) + do_test_OffsetCommit_automatic_stale_member_epoch_error( + mcluster, bootstraps, i); + + do_test_consumer_inflight_heartbeat_on_leave(mcluster, bootstraps); + + test_mock_cluster_destroy(mcluster); + + return 0; +} diff --git a/lib/librdkafka-2.10.1/tests/0149-broker-same-host-port.c b/lib/librdkafka-2.15.0/tests/0149-broker-same-host-port.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0149-broker-same-host-port.c rename to lib/librdkafka-2.15.0/tests/0149-broker-same-host-port.c diff --git a/lib/librdkafka-2.10.1/tests/0150-telemetry_mock.c b/lib/librdkafka-2.15.0/tests/0150-telemetry_mock.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0150-telemetry_mock.c rename to lib/librdkafka-2.15.0/tests/0150-telemetry_mock.c diff --git a/lib/librdkafka-2.10.1/tests/0151-purge-brokers.c b/lib/librdkafka-2.15.0/tests/0151-purge-brokers.c similarity index 96% rename from lib/librdkafka-2.10.1/tests/0151-purge-brokers.c rename to lib/librdkafka-2.15.0/tests/0151-purge-brokers.c index c36a526c8c3..4728863e2fb 100644 --- a/lib/librdkafka-2.10.1/tests/0151-purge-brokers.c +++ b/lib/librdkafka-2.15.0/tests/0151-purge-brokers.c @@ -140,7 +140,6 @@ static void fetch_metadata(rd_kafka_t *rk, * action. * @param await_verification If `rd_false`, the verification is * done only after last action. - * @return The opaque set in the `rd_kafka_t` handle. */ #define TEST_ACTION_REMOVE_BROKER 0 #define TEST_ACTION_ADD_BROKER 1 @@ -148,7 +147,7 @@ static void fetch_metadata(rd_kafka_t *rk, #define TEST_ACTION_SET_UP_BROKER 3 #define TEST_ACTION_SET_GROUP_COORDINATOR 4 #define TEST_GROUP "topic1" -static void *do_test_add_remove_brokers0( +static void do_test_add_remove_brokers0( int32_t initial_cluster_size, int32_t actions[][2], size_t action_cnt, @@ -162,7 +161,6 @@ static void *do_test_add_remove_brokers0( rd_kafka_conf_t *conf; rd_kafka_t *rk; size_t action = 0; - void *opaque; cluster = test_mock_cluster_new(initial_cluster_size, &bootstraps); @@ -247,10 +245,8 @@ static void *do_test_add_remove_brokers0( TEST_SAY("Test verification complete\n"); rd_atomic32_set(&verification_complete, 1); - opaque = rd_kafka_opaque(rk); rd_kafka_destroy(rk); test_mock_cluster_destroy(cluster); - return opaque; } /** @@ -391,7 +387,6 @@ static void do_test_remove_then_add(void) { SUB_TEST_QUICK(); rd_atomic32_init(&do_test_remove_then_add_received_terminate, 0); rd_atomic32_init(&verification_complete, 0); - test_conf_log_interceptor_t *log_interceptor; int32_t expected_brokers_cnt[] = {3, 3, 2, 3}; @@ -404,12 +399,13 @@ static void do_test_remove_then_add(void) { {TEST_ACTION_ADD_BROKER, 1}, }; - log_interceptor = do_test_add_remove_brokers0( + do_test_add_remove_brokers0( 3, actions, RD_ARRAY_SIZE(actions), expected_broker_ids, expected_brokers_cnt, do_test_remove_then_add_edit_configuration_cb, NULL, do_test_remove_then_add_await_after_action_cb); rd_free(log_interceptor); + log_interceptor = NULL; SUB_TEST_PASS(); } @@ -461,7 +457,7 @@ do_test_down_then_up_no_rebootstrap_loop_request_metadata_cb(int action) { static rd_bool_t do_test_down_then_up_no_rebootstrap_loop_await_after_action_cb(int action) { if (action == 1) { - rd_sleep(5); + rd_sleep(6); } return rd_false; } @@ -493,18 +489,24 @@ static void do_test_down_then_up_no_rebootstrap_loop(void) { do_test_down_then_up_no_rebootstrap_loop_request_metadata_cb, do_test_down_then_up_no_rebootstrap_loop_await_after_action_cb); - /* With connections that go always to the broker without previous - * connections (the re-bootstrapped one) we get 5 re-bootstrap - * sequences. Given a 90% probability of selecting the learned broker - * there's a 10% probability of selecting the bootstrap one. - * The expected value is 0.5, we expect <= 3 here. */ + /* With a rebootstrap every time the bootstrap brokers are removed + * we get 6 re-bootstrap sequences. + * With the fix we require connection to all learned brokers before + * reaching all brokers down again. + * In this case we have to connect to the bootstrap broker + * and the learned broker, 2s in the slowest case as it depends + * on periodic 10s brokers refresh timer too. + * We expect 5 or less re-bootstrap sequences. */ TEST_ASSERT( rd_atomic32_get( &do_test_down_then_up_no_rebootstrap_loop_rebootstrap_sequence_cnt) <= - 3, - "Expected <= 3 re-bootstrap sequences, got %d", + 5, + "Expected <= 5 re-bootstrap sequences, got %d", rd_atomic32_get( &do_test_down_then_up_no_rebootstrap_loop_rebootstrap_sequence_cnt)); + + rd_free(log_interceptor); + log_interceptor = NULL; SUB_TEST_PASS(); } diff --git a/lib/librdkafka-2.10.1/tests/0152-rebootstrap.c b/lib/librdkafka-2.15.0/tests/0152-rebootstrap.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/0152-rebootstrap.c rename to lib/librdkafka-2.15.0/tests/0152-rebootstrap.c diff --git a/lib/librdkafka-2.15.0/tests/0153-memberid.c b/lib/librdkafka-2.15.0/tests/0153-memberid.c new file mode 100644 index 00000000000..56691c02bde --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0153-memberid.c @@ -0,0 +1,128 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +typedef struct consumer_s { + const char *group_id; + char *memberid; +} consumer_t; + +static int +is_fatal_cb(rd_kafka_t *rk, rd_kafka_resp_err_t err, const char *reason) { + if (err == RD_KAFKA_RESP_ERR__TIMED_OUT) + return 0; + return 1; +} + +static int consumer_thread(void *arg) { + rd_kafka_conf_t *conf; + rd_kafka_t *consumer; + consumer_t *consumer_args = arg; + + test_curr->is_fatal_cb = is_fatal_cb; + + test_conf_init(&conf, NULL, 60); + + consumer = + test_create_consumer(consumer_args->group_id, NULL, conf, NULL); + + consumer_args->memberid = rd_kafka_memberid(consumer); + + test_consumer_close(consumer); + rd_kafka_destroy(consumer); + test_curr->is_fatal_cb = NULL; + return 0; +} + +void do_test_unique_memberid() { + int i; + int j; + int have_only_unique_memberids = 1; + const char *group_id = test_mk_topic_name("0153-memberid", 1); + +#define CONSUMER_CNT 500 + thrd_t thread_id[CONSUMER_CNT]; + consumer_t consumer_args[CONSUMER_CNT]; + + SUB_TEST_QUICK(); + + for (i = 0; i < CONSUMER_CNT; i++) { + consumer_args[i].group_id = group_id; + consumer_args[i].memberid = NULL; + thrd_create(&thread_id[i], consumer_thread, &consumer_args[i]); + } + + for (i = 0; i < CONSUMER_CNT; i++) { + thrd_join(thread_id[i], NULL); + } + + for (i = 0; i < CONSUMER_CNT; i++) { + if (have_only_unique_memberids) { + for (j = i + 1; j < CONSUMER_CNT; j++) { + if (strcmp(consumer_args[i].memberid, + consumer_args[j].memberid) == 0) { + TEST_SAY( + "Consumer %d has the same member " + "ID as consumer %d: %s\n", + i, j, consumer_args[i].memberid); + have_only_unique_memberids = 0; + break; + } + } + } + rd_free(consumer_args[i].memberid); + } + + if (have_only_unique_memberids) { + TEST_SAY("All %d consumers have unique member IDs\n", + CONSUMER_CNT); + } else { + TEST_FAIL("Not all consumers have unique member IDs\n"); + } + + SUB_TEST_PASS(); +} + +int main_0153_memberid(int argc, char **argv) { + + if (test_consumer_group_protocol_classic()) { + TEST_SKIP( + "Member ID is not generated on the client side in classic " + "protocol, skipping test"); + return 0; + } + + if (!strcmp(test_mode, "valgrind")) { + TEST_SKIP("Test is too heavy for valgrind, skipping it"); + return 0; + } + + do_test_unique_memberid(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0155-share_group_heartbeat_mock.c b/lib/librdkafka-2.15.0/tests/0155-share_group_heartbeat_mock.c new file mode 100644 index 00000000000..191f9960bea --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0155-share_group_heartbeat_mock.c @@ -0,0 +1,2695 @@ +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @name Mock tests for share consumer and ShareGroupHeartbeat + * + * Note: rd_kafka_assignment() and rd_kafka_fatal_error() are called via + * test_share_consumer_get_rk() to access the underlying rd_kafka_t + * handle from the share consumer. + */ + +static rd_bool_t is_share_heartbeat_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == + RD_KAFKAP_ShareGroupHeartbeat; +} + +/** + * @brief Wait for at least \p num ShareGroupHeartbeat requests + * to be received by the mock cluster. + * + * @return Number of heartbeats received. + */ +static int wait_share_heartbeats(rd_kafka_mock_cluster_t *mcluster, + int num, + int confidence_interval) { + return test_mock_wait_matching_requests( + mcluster, num, confidence_interval, is_share_heartbeat_request, + NULL); +} + +/** + * @brief Create a share consumer connected to mock cluster. + */ +static rd_kafka_share_t *create_share_consumer(const char *bootstraps, + const char *group_id) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer: %s", + errstr); + + return rkshare; +} + +/** + * @brief Poll rd_kafka_share_poll() until it returns a fatal + * error or \p timeout_ms elapses. + * + * While waiting for the fatal error no records are expected, and any + * error returned must be fatal; both are asserted. + * + * @return The fatal rd_kafka_error_t* (caller owns it and must destroy it), + * or NULL if the timeout elapsed without any error. + */ +static rd_kafka_error_t *wait_fatal_error(rd_kafka_share_t *share_c, + int timeout_ms) { + int64_t deadline = test_clock() + (int64_t)timeout_ms * 1000; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + rd_kafka_error_t *error; + + while (test_clock() < deadline) { + error = rd_kafka_share_poll(share_c, 100, &batch); + rcvd = rd_kafka_messages_count(batch); + + TEST_ASSERT(rcvd == 0, + "Expected no records while waiting for fatal " + "error, got %d", + (int)rcvd); + + rd_kafka_messages_destroy(batch); + batch = NULL; + + if (error) { + TEST_ASSERT(rd_kafka_error_is_fatal(error), + "Expected a fatal error, got non-fatal %s", + rd_kafka_error_name(error)); + return error; + } + } + return NULL; +} + +static int count_topic_partitions(rd_kafka_topic_partition_list_t *assignment, + const char *topic) { + int i, count = 0; + for (i = 0; i < assignment->cnt; i++) { + if (strcmp(assignment->elems[i].topic, topic) == 0) + count++; + } + return count; +} + +/** + * @brief Wait until rd_kafka_assignment() returns exactly + * \p expected_cnt partitions, or \p timeout_ms elapses. + * + * The heartbeat thread updates assignments independently, + * so no polling is needed. + * + * @return The final assignment count. + */ +static int wait_assignment_count(rd_kafka_share_t *share_c, + int expected_cnt, + int timeout_ms) { + int64_t deadline = test_clock() + (int64_t)timeout_ms * 1000; + int cnt = -1; + + while (test_clock() < deadline) { + rd_kafka_topic_partition_list_t *assignment; + + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c), &assignment)); + cnt = assignment->cnt; + rd_kafka_topic_partition_list_destroy(assignment); + + if (cnt == expected_cnt) + return cnt; + + rd_usleep(500 * 1000, 0); + } + return cnt; +} + + +/** + * @brief Test basic ShareGroupHeartbeat flow: + * join, receive assignment, heartbeats, leave. + */ +static void do_test_share_group_heartbeat_basic(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + int found_heartbeats, cnt; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for join heartbeat */ + found_heartbeats = wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(found_heartbeats >= 1, + "Expected at least 1 heartbeat, got %d", found_heartbeats); + + /* Wait until assignment propagates */ + cnt = wait_assignment_count(share_c, 3, 10000); + TEST_ASSERT(cnt == 3, "Expected 3 partitions assigned, got %d", cnt); + + /* Verify multiple heartbeats */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT(found_heartbeats >= 2, + "Expected at least 2 heartbeats, got %d", found_heartbeats); + + /* Close consumer (sends leave heartbeat) */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + /* Verify leave heartbeat was sent */ + found_heartbeats = wait_share_heartbeats(mcluster, 3, 1000); + + /* Verify no more heartbeats after leave. + * Use a generous sleep (5s) and confidence interval (1000ms) + * to avoid false positives under CPU contention. */ + rd_kafka_mock_stop_request_tracking(mcluster); + rd_kafka_mock_start_request_tracking(mcluster); + rd_sleep(5); + found_heartbeats = wait_share_heartbeats(mcluster, 0, 1000); + TEST_ASSERT(found_heartbeats == 0, + "Expected 0 heartbeats after leave, got %d", + found_heartbeats); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test assignment redistribution when consumers join/leave. + */ +static void do_test_share_group_assignment_rebalance(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_topic_partition_list_t *share_c1_assignment, + *share_c2_assignment; + rd_kafka_share_t *share_c1, *share_c2; + int64_t deadline; + int cnt; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-rebalance"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c1 = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + + /* C1 joins - wait for all 3 partitions */ + cnt = wait_assignment_count(share_c1, 3, 10000); + TEST_ASSERT(cnt == 3, "Expected C1 to have 3 partitions, got %d", cnt); + + /* C2 joins - partitions should be redistributed */ + share_c2 = create_share_consumer(bootstraps, group); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait until both consumers have partitions and total == 3 */ + deadline = test_clock() + 15000 * 1000; + while (test_clock() < deadline) { + + TEST_CALL_ERR__( + rd_kafka_assignment(test_share_consumer_get_rk(share_c1), + &share_c1_assignment)); + TEST_CALL_ERR__( + rd_kafka_assignment(test_share_consumer_get_rk(share_c2), + &share_c2_assignment)); + + if (share_c1_assignment->cnt + share_c2_assignment->cnt == 3 && + share_c1_assignment->cnt > 0 && + share_c2_assignment->cnt > 0) { + rd_kafka_topic_partition_list_destroy( + share_c1_assignment); + rd_kafka_topic_partition_list_destroy( + share_c2_assignment); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assignment); + rd_kafka_topic_partition_list_destroy(share_c2_assignment); + rd_usleep(200 * 1000, 0); + } + /* Final check after loop */ + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assignment)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assignment)); + TEST_ASSERT(share_c1_assignment->cnt + share_c2_assignment->cnt == 3, + "Expected total 3 partitions, got %d + %d = %d", + share_c1_assignment->cnt, share_c2_assignment->cnt, + share_c1_assignment->cnt + share_c2_assignment->cnt); + TEST_ASSERT(share_c1_assignment->cnt > 0 && + share_c2_assignment->cnt > 0, + "Expected both consumers to have partitions, " + "got C1=%d, C2=%d", + share_c1_assignment->cnt, share_c2_assignment->cnt); + rd_kafka_topic_partition_list_destroy(share_c1_assignment); + rd_kafka_topic_partition_list_destroy(share_c2_assignment); + + /* C2 leaves - C1 should get all partitions back */ + test_share_consumer_close(share_c2); + test_share_destroy(share_c2); + + cnt = wait_assignment_count(share_c1, 3, 10000); + TEST_ASSERT(cnt == 3, + "Expected C1 to have 3 partitions after C2 left, got %d", + cnt); + + /* Cleanup */ + test_share_consumer_close(share_c1); + test_share_destroy(share_c1); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test multi-topic assignment with mixed subscriptions. + * C1: both topics, C2: orders only, C3: events only + */ +static void do_test_share_group_multi_topic_assignment(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *sub_both, *sub_orders, *sub_events; + rd_kafka_topic_partition_list_t *share_c1_assign, *share_c2_assign, + *share_c3_assign; + rd_kafka_share_t *share_c1, *share_c2, *share_c3; + const char *topic_orders = "test-orders"; + const char *topic_events = "test-events"; + const char *group = "test-share-group-multi"; + int total_orders, total_events, cnt; + int64_t deadline; + + SUB_TEST_QUICK(); + + /* Setup: orders (4 partitions), events (2 partitions) */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic_orders, 4, 1); + rd_kafka_mock_topic_create(mcluster, topic_events, 2, 1); + + sub_both = rd_kafka_topic_partition_list_new(2); + rd_kafka_topic_partition_list_add(sub_both, topic_orders, + RD_KAFKA_PARTITION_UA); + rd_kafka_topic_partition_list_add(sub_both, topic_events, + RD_KAFKA_PARTITION_UA); + + sub_orders = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(sub_orders, topic_orders, + RD_KAFKA_PARTITION_UA); + + sub_events = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(sub_events, topic_events, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + + /* C1 joins (both topics) - should get all 6 partitions */ + share_c1 = create_share_consumer(bootstraps, group); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, sub_both)); + cnt = wait_assignment_count(share_c1, 6, 10000); + TEST_ASSERT(cnt == 6, "C1 should have all 6 partitions, got %d", cnt); + + /* C2 joins (orders only) - orders should split. + * Wait until C2 has at least 1 orders partition and + * total orders == 4, total events == 2. */ + share_c2 = create_share_consumer(bootstraps, group); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, sub_orders)); + + deadline = test_clock() + 15000 * 1000; + while (test_clock() < deadline) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + + total_orders = + count_topic_partitions(share_c1_assign, topic_orders) + + count_topic_partitions(share_c2_assign, topic_orders); + total_events = + count_topic_partitions(share_c1_assign, topic_events) + + count_topic_partitions(share_c2_assign, topic_events); + + if (total_orders == 4 && total_events == 2 && + count_topic_partitions(share_c2_assign, topic_orders) > 0) { + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_usleep(200 * 1000, 0); + } + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + total_orders = count_topic_partitions(share_c1_assign, topic_orders) + + count_topic_partitions(share_c2_assign, topic_orders); + total_events = count_topic_partitions(share_c1_assign, topic_events) + + count_topic_partitions(share_c2_assign, topic_events); + TEST_ASSERT(total_orders == 4, "Total orders should be 4, got %d", + total_orders); + TEST_ASSERT(total_events == 2, "Total events should be 2, got %d", + total_events); + TEST_ASSERT(count_topic_partitions(share_c2_assign, topic_orders) > 0, + "C2 should have at least 1 orders partition"); + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + + /* C3 joins (events only) - events should split. + * Wait until C3 has at least 1 events partition. */ + share_c3 = create_share_consumer(bootstraps, group); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c3, sub_events)); + + deadline = test_clock() + 15000 * 1000; + while (test_clock() < deadline) { + + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c3), &share_c3_assign)); + + total_orders = + count_topic_partitions(share_c1_assign, topic_orders) + + count_topic_partitions(share_c2_assign, topic_orders) + + count_topic_partitions(share_c3_assign, topic_orders); + total_events = + count_topic_partitions(share_c1_assign, topic_events) + + count_topic_partitions(share_c2_assign, topic_events) + + count_topic_partitions(share_c3_assign, topic_events); + + if (total_orders == 4 && total_events == 2 && + count_topic_partitions(share_c3_assign, topic_events) > 0) { + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + rd_usleep(200 * 1000, 0); + } + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c3), &share_c3_assign)); + total_orders = count_topic_partitions(share_c1_assign, topic_orders) + + count_topic_partitions(share_c2_assign, topic_orders) + + count_topic_partitions(share_c3_assign, topic_orders); + total_events = count_topic_partitions(share_c1_assign, topic_events) + + count_topic_partitions(share_c2_assign, topic_events) + + count_topic_partitions(share_c3_assign, topic_events); + TEST_ASSERT(total_orders == 4, "Total orders should be 4, got %d", + total_orders); + TEST_ASSERT(total_events == 2, "Total events should be 2, got %d", + total_events); + TEST_ASSERT(count_topic_partitions(share_c3_assign, topic_events) > 0, + "C3 should have at least 1 events partition"); + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + + /* C1 leaves - C2 should get all orders, C3 all events. + * Wait until C2 has 4 orders and C3 has 2 events. */ + test_share_consumer_close(share_c1); + test_share_destroy(share_c1); + + deadline = test_clock() + 15000 * 1000; + while (test_clock() < deadline) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c3), &share_c3_assign)); + + if (count_topic_partitions(share_c2_assign, topic_orders) == + 4 && + count_topic_partitions(share_c3_assign, topic_events) == + 2) { + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + break; + } + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + rd_usleep(200 * 1000, 0); + } + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c3), &share_c3_assign)); + TEST_ASSERT(count_topic_partitions(share_c2_assign, topic_orders) == 4, + "C2 should have all 4 orders partitions, got %d", + count_topic_partitions(share_c2_assign, topic_orders)); + TEST_ASSERT(count_topic_partitions(share_c3_assign, topic_events) == 2, + "C3 should have all 2 events partitions, got %d", + count_topic_partitions(share_c3_assign, topic_events)); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + + /* C2 leaves - C3 should still have events only */ + test_share_consumer_close(share_c2); + test_share_destroy(share_c2); + + /* Wait for C3 to stabilize with 2 events, 0 orders */ + cnt = wait_assignment_count(share_c3, 2, 10000); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c3), &share_c3_assign)); + TEST_ASSERT(count_topic_partitions(share_c3_assign, topic_events) == 2, + "C3 should still have 2 events partitions, got %d", + count_topic_partitions(share_c3_assign, topic_events)); + TEST_ASSERT(count_topic_partitions(share_c3_assign, topic_orders) == 0, + "C3 should have 0 orders partitions (not subscribed), " + "got %d", + count_topic_partitions(share_c3_assign, topic_orders)); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + + /* Cleanup */ + test_share_consumer_close(share_c3); + test_share_destroy(share_c3); + + rd_kafka_topic_partition_list_destroy(sub_both); + rd_kafka_topic_partition_list_destroy(sub_orders); + rd_kafka_topic_partition_list_destroy(sub_events); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test fatal error injection for ShareGroupHeartbeat. + * + * When a fatal exception occurs during heartbeat, the consumer should + * transition to fatal state and no longer be usable. + */ +static void do_test_share_group_error_injection(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-errors"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject a fatal error (INVALID_REQUEST) during heartbeat. + * This matches testFailureOnFatalException which verifies + * transitionToFatal() is called on fatal heartbeat errors. */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_INVALID_REQUEST, 0); + + /* Wait for the fatal error to propagate via consume_batch. */ + error = wait_fatal_error(share_c, 7000); + TEST_ASSERT(error != NULL, + "Expected a fatal error but none received within timeout"); + TEST_ASSERT(rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR_INVALID_REQUEST, + "Expected INVALID_REQUEST fatal error, got %s", + rd_kafka_error_name(error)); + TEST_SAY("Consumer entered fatal state: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + + /* Cleanup. Consumer is in fatal state, but close() still flushes + * pending acks and leaves the share session, so it succeeds. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test network timeout for ShareGroupHeartbeat. + * + * When a heartbeat times out due to network latency, the consumer should + * handle the timeout and retry with backoff, eventually recovering. + */ +static void do_test_share_group_rtt_injection(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + rd_kafka_conf_t *conf; + char errstr[512]; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-rtt"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + /* Create consumer with short socket timeout so RTT injection + * causes an actual timeout. Default is 60s which is too long. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "socket.timeout.ms", "3000"); + + share_c = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(share_c != NULL, "Failed to create share consumer: %s", + errstr); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 500); + rd_usleep(500 * 1000, 0); + + /* Verify initial assignment */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Expected 3 partitions initially, got %d", assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Inject RTT larger than socket.timeout.ms to cause a real timeout. + * The Java test verifies TimeoutException + backoff retry. */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, 5000); + + /* Wait through the timeout period - consumer should recover */ + rd_usleep(500 * 1000, 0); + + /* Verify heartbeats resumed after timeout recovery */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT(found_heartbeats >= 1, + "Expected heartbeats to resume after timeout, got %d", + found_heartbeats); + + /* Wait for assignment to be restored */ + rd_usleep(500 * 1000, 0); + + /* Verify consumer recovered and still has assignment */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Expected 3 partitions after timeout recovery, got %d", + assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test session timeout for ShareGroupHeartbeat. + * + * Tests that the mock broker correctly times out members that stop + * heartbeating. Uses a short session timeout (3000ms) and verifies: + * - Member is removed after timeout + * - Remaining members get reassigned partitions + */ +static void do_test_share_group_session_timeout(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_topic_partition_list_t *share_c1_assign, *share_c2_assign; + rd_kafka_share_t *share_c1, *share_c2; + int share_c1_initial = 0, share_c2_initial = 0; + int64_t dl; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-timeout"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 4, 1); + + /* Set heartbeat interval shorter than session timeout so consumers + * don't time out while waiting for assignment updates. */ + rd_kafka_mock_sharegroup_set_heartbeat_interval(mcluster, 1000); + /* Set session timeout. Must be > heartbeat_interval to avoid + * spurious timeouts but short enough for the test to finish + * quickly. */ + rd_kafka_mock_sharegroup_set_session_timeout(mcluster, 3000); + + share_c1 = create_share_consumer(bootstraps, group); + share_c2 = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for both to join and rebalance to complete. */ + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + share_c1_initial = share_c1_assign->cnt; + share_c2_initial = share_c2_assign->cnt; + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + if (share_c1_initial + share_c2_initial == 4 && + share_c1_initial > 0 && share_c2_initial > 0) + break; + rd_usleep(200 * 1000, 0); + } + TEST_ASSERT(share_c1_initial + share_c2_initial == 4, + "Total should be 4 partitions, got %d", + share_c1_initial + share_c2_initial); + TEST_ASSERT(share_c1_initial > 0 && share_c2_initial > 0, + "Both consumers should have partitions"); + + /* Destroy C2 without close to simulate crash */ + test_share_destroy(share_c2); + + /* Wait for C2's session to time out (3s) and C1 to get + * all partitions back. */ + TEST_ASSERT(wait_assignment_count(share_c1, 4, 10000) == 4, + "C1 should have all 4 partitions after C2 timeout"); + + test_share_consumer_close(share_c1); + test_share_destroy(share_c1); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test target assignment API for ShareGroupHeartbeat. + * + * Tests that the mock broker can apply manual target assignments: + * 1. Two consumers join and get automatic assignment (2 partitions each) + * 2. Retrieve member IDs using rd_kafka_mock_sharegroup_get_member_ids() + * 3. Set manual target assignment: C1 gets all 4 partitions, C2 gets none + * 4. Verify consumers receive the manual assignment + */ +static void do_test_share_group_target_assignment(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_topic_partition_list_t *share_c1_assign, *share_c2_assign; + rd_kafka_topic_partition_list_t *target_c1, *target_c2; + rd_kafka_topic_partition_list_t *assignments[2]; + rd_kafka_share_t *share_c1, *share_c2; + char **member_ids; + size_t member_cnt; + rd_kafka_resp_err_t err; + int64_t dl; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-target"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 4, 1); + + share_c1 = create_share_consumer(bootstraps, group); + share_c2 = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for both to join and rebalance to complete */ + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + if (share_c1_assign->cnt + share_c2_assign->cnt == 4 && + share_c1_assign->cnt > 0 && share_c2_assign->cnt > 0) { + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_usleep(200 * 1000, 0); + } + /* Final check */ + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_ASSERT(share_c1_assign->cnt + share_c2_assign->cnt == 4, + "Total should be 4 partitions, got %d", + share_c1_assign->cnt + share_c2_assign->cnt); + TEST_ASSERT(share_c1_assign->cnt > 0 && share_c2_assign->cnt > 0, + "Both consumers should have partitions initially"); + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + + /* Retrieve member IDs */ + err = rd_kafka_mock_sharegroup_get_member_ids(mcluster, group, + &member_ids, &member_cnt); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected no error, got %s", rd_kafka_err2str(err)); + TEST_ASSERT(member_cnt == 2, "Expected 2 members, got %zu", member_cnt); + + /* Set manual target assignment: all to first member */ + target_c1 = rd_kafka_topic_partition_list_new(4); + rd_kafka_topic_partition_list_add(target_c1, topic, 0); + rd_kafka_topic_partition_list_add(target_c1, topic, 1); + rd_kafka_topic_partition_list_add(target_c1, topic, 2); + rd_kafka_topic_partition_list_add(target_c1, topic, 3); + + target_c2 = rd_kafka_topic_partition_list_new(0); + + assignments[0] = target_c1; + assignments[1] = target_c2; + + rd_kafka_mock_sharegroup_target_assignment( + mcluster, group, (const char **)member_ids, assignments, 2); + + rd_kafka_topic_partition_list_destroy(target_c1); + rd_kafka_topic_partition_list_destroy(target_c2); + + /* Wait until one consumer has all 4 and the other has 0. */ + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + + if ((share_c1_assign->cnt == 4 && share_c2_assign->cnt == 0) || + (share_c1_assign->cnt == 0 && share_c2_assign->cnt == 4)) { + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_usleep(200 * 1000, 0); + } + + /* Verify manual assignment was applied */ + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + + TEST_ASSERT(share_c1_assign->cnt + share_c2_assign->cnt == 4, + "Total should still be 4 partitions, got %d", + share_c1_assign->cnt + share_c2_assign->cnt); + TEST_ASSERT( + (share_c1_assign->cnt == 4 && share_c2_assign->cnt == 0) || + (share_c1_assign->cnt == 0 && share_c2_assign->cnt == 4), + "Expected one consumer to have all 4 partitions and the " + "other to have 0, got C1=%d, C2=%d", + share_c1_assign->cnt, share_c2_assign->cnt); + + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + + /* Free member IDs */ + rd_free(member_ids[0]); + rd_free(member_ids[1]); + rd_free(member_ids); + + /* Cleanup */ + test_share_consumer_close(share_c1); + test_share_consumer_close(share_c2); + test_share_destroy(share_c1); + test_share_destroy(share_c2); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that a healthy, continuously-heartbeating member is never + * fenced by the mock broker's session timeout timer. + * + * This test sets a 2-second session timeout and then keeps the consumer + * active for 10 seconds. If the session timeout timer incorrectly fences + * active members, the assignment would drop. This validates that the timer + * only fences truly timed-out members (those that stopped heartbeating). + * Related to KIP-932 session timeout logic. + */ +static void do_test_share_group_no_spurious_fencing(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-no-fence"; + int i; + + SUB_TEST_QUICK(); + + /* Setup with a short session timeout and a heartbeat interval that + * is well below it, so the active consumer is never spuriously + * timed out. */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + rd_kafka_mock_sharegroup_set_heartbeat_interval(mcluster, 500); + rd_kafka_mock_sharegroup_set_session_timeout(mcluster, 2000); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for join and initial assignment. */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Wait for 5s (2.5x the 2s session timeout) and verify + * the assignment is not dropped. + * If the broker's session timeout timer incorrectly fences active + * members, the assignment will drop. */ + TEST_SAY("Waiting for 5 seconds with 2s session timeout...\n"); + for (i = 0; i < 5; i++) { + rd_usleep(1000 * 1000, 0); + + /* Verify assignment is still intact */ + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c), &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Assignment dropped at %ds (spurious fencing!)", + i + 1); + rd_kafka_topic_partition_list_destroy(assignment); + } + + TEST_SAY("No spurious fencing after 5 seconds\n"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/* TODO: Add do_test_share_group_max_size() once + * 0155 fix PR is merged*/ + +/** + * @brief UNKNOWN_MEMBER_ID error handling. + * + * When a consumer receives UNKNOWN_MEMBER_ID error, it should rejoin + * with epoch=0 (fresh join). + * + * NOT YET COMPATIBLE: UNKNOWN_MEMBER_ID triggers an incorrect assert in + * development builds (rdkafka_cgrp.c:6631). Pending fix by Pranav. + * See sghb_test_discrepancies.txt #3. + */ +static void do_test_unknown_member_id_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-unknown-member"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject UNKNOWN_MEMBER_ID error */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID, 0); + + /* Wait for consumer to handle error and rejoin */ + rd_usleep(500 * 1000, 0); + + /* Verify heartbeats continue (rejoin happened) */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT(found_heartbeats >= 1, + "Expected heartbeats to continue after UNKNOWN_MEMBER_ID, " + "got %d", + found_heartbeats); + + /* Verify consumer eventually gets assignment back */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after rejoin"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief FENCED_MEMBER_EPOCH error handling. + * + * When a consumer receives FENCED_MEMBER_EPOCH error, it should be fenced + * and then rejoin with epoch=0. + */ +static void do_test_fenced_member_epoch_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-fenced"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject FENCED_MEMBER_EPOCH error */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_FENCED_MEMBER_EPOCH, 0); + + /* Wait for consumer to handle error and rejoin */ + rd_usleep(500 * 1000, 0); + + /* Verify heartbeats continue (rejoin happened) */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT( + found_heartbeats >= 1, + "Expected heartbeats to continue after FENCED_MEMBER_EPOCH, " + "got %d", + found_heartbeats); + + /* Verify consumer eventually gets assignment back */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after rejoin"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief COORDINATOR_NOT_AVAILABLE error handling. + * + * When a consumer receives COORDINATOR_NOT_AVAILABLE, it should retry + * (retriable error). + */ +static void do_test_coordinator_not_available_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-coord-unavail"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject COORDINATOR_NOT_AVAILABLE error (transient) */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, 0); + + /* Wait for consumer to handle transient error and retry */ + rd_usleep(500 * 1000, 0); + + /* Verify heartbeats continue after transient error */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT( + found_heartbeats >= 1, + "Expected heartbeats to continue after COORDINATOR_NOT_AVAILABLE, " + "got %d", + found_heartbeats); + + /* Verify consumer still has assignment */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after retry"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief NOT_COORDINATOR error handling. + * + * When a consumer receives NOT_COORDINATOR, it should find a new coordinator. + */ +static void do_test_not_coordinator_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-not-coord"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject NOT_COORDINATOR error */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_NOT_COORDINATOR, 0); + + /* Wait for consumer to find new coordinator and continue. + * NOT_COORDINATOR triggers coordinator rediscovery which may take + * longer than COORDINATOR_NOT_AVAILABLE. */ + rd_usleep(500 * 1000, 0); + + /* Verify heartbeats continue after finding coordinator */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT(found_heartbeats >= 1, + "Expected heartbeats to continue after NOT_COORDINATOR, " + "got %d", + found_heartbeats); + + /* Verify consumer still has assignment */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after finding coordinator"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief GROUP_AUTHORIZATION_FAILED error handling (fatal). + * + * When a consumer receives GROUP_AUTHORIZATION_FAILED, it should treat + * it as a fatal error. + */ +static void do_test_group_authorization_failed_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-auth-failed"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join */ + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + /* Inject GROUP_AUTHORIZATION_FAILED error (fatal) */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED, 0); + + /* Wait for the fatal error to propagate via consume_batch. */ + error = wait_fatal_error(share_c, 5000); + TEST_ASSERT(error != NULL, + "Expected a fatal error but none received within timeout"); + TEST_ASSERT(rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED, + "Expected GROUP_AUTHORIZATION_FAILED fatal error, got %s", + rd_kafka_error_name(error)); + TEST_SAY("Consumer entered fatal state: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + + /* Cleanup. Consumer is in fatal state, but close() still flushes + * pending acks and leaves the share session, so it succeeds. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief GROUP_MAX_SIZE_REACHED error handling. + * + * When a new member tries to join and gets GROUP_MAX_SIZE_REACHED, + * the error should be treated as fatal for that consumer. + */ +static void do_test_group_max_size_reached_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c1, *share_c2; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-max-size"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 4, 1); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + /* First consumer joins successfully */ + share_c1 = create_share_consumer(bootstraps, group); + + rd_kafka_mock_start_request_tracking(mcluster); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + + /* Wait for share_c1 to fully join and stabilize */ + TEST_ASSERT(wait_assignment_count(share_c1, 4, 10000) == 4, + "Expected share_c1 to have 4 partitions"); + + /* Push multiple GROUP_MAX_SIZE_REACHED errors so that even if + * share_c1's regular heartbeat consumes some, share_c2's join heartbeat + * will also get one. The Java test uses server-side maxSize=1 + * config; we simulate by injecting errors for all heartbeats. */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 5, + RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED, 0); + + /* Create second consumer - should be rejected */ + share_c2 = create_share_consumer(bootstraps, group); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + + /* Wait for the fatal error to propagate via consume_batch. */ + error = wait_fatal_error(share_c2, 5000); + TEST_ASSERT(error != NULL, + "Expected a fatal error but none received within timeout"); + TEST_ASSERT(rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED, + "Expected GROUP_MAX_SIZE_REACHED fatal error, got %s", + rd_kafka_error_name(error)); + TEST_SAY("share consumer 2 correctly rejected with fatal error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + + rd_kafka_topic_partition_list_destroy(subscription); + + /* Cleanup. share_c2 is in fatal state, but close() no longer + * early-returns the stored fatal error, so it succeeds. */ + test_share_consumer_close(share_c1); + rd_kafka_error_t *c2_close_error = + rd_kafka_share_consumer_close(share_c2); + TEST_ASSERT(!c2_close_error, + "Expected share_c2 close to succeed, " + "got %s", + rd_kafka_error_name(c2_close_error)); + test_share_destroy(share_c1); + test_share_destroy(share_c2); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Member rejoin with epoch zero. + * + * A member in stable state (epoch > 0) that sends heartbeat with epoch=0 + * should be treated as a rejoin and assigned a new member ID. + */ +static void do_test_member_rejoin_with_epoch_zero(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-rejoin"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + /* Verify initial assignment (member is now in stable state) */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Expected 3 partitions initially, got %d", assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Force a rejoin by injecting UNKNOWN_MEMBER_ID error + * This will cause client to rejoin with epoch=0 */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID, 0); + + /* Wait for consumer to rejoin with epoch=0 */ + rd_usleep(2000 * 1000, 0); + + /* Verify rejoin heartbeats */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT(found_heartbeats >= 1, "Expected rejoin heartbeats, got %d", + found_heartbeats); + + /* Verify consumer gets assignment back */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after rejoin"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Leaving member bumps group epoch. + * + * When a member sends leave heartbeat (epoch=-1), the group epoch should + * be bumped and remaining members should get updated assignment. + */ +static void do_test_leaving_member_bumps_group_epoch(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_topic_partition_list_t *share_c1_assign, *share_c2_assign; + rd_kafka_share_t *share_c1, *share_c2; + int64_t dl; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-leave-epoch"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 4, 1); + + /* Set heartbeat interval shorter than session timeout so both + * consumers can heartbeat frequently enough for the rebalance + * to complete before we check assignments. */ + rd_kafka_mock_sharegroup_set_heartbeat_interval(mcluster, 1000); + + share_c1 = create_share_consumer(bootstraps, group); + share_c2 = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for both to join and rebalance to complete */ + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + if (share_c1_assign->cnt + share_c2_assign->cnt == 4 && + share_c1_assign->cnt > 0 && share_c2_assign->cnt > 0) { + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_usleep(200 * 1000, 0); + } + + /* C2 leaves (sends epoch=-1 leave heartbeat) */ + test_share_consumer_close(share_c2); + test_share_destroy(share_c2); + + /* Wait for C1 to get all partitions after C2 left */ + TEST_ASSERT(wait_assignment_count(share_c1, 4, 10000) == 4, + "C1 should have all 4 partitions after C2 left"); + + /* Cleanup */ + test_share_consumer_close(share_c1); + test_share_destroy(share_c1); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Partition assignment with changing topics. + * + * Note: This test is limited in mock broker - we can test initial assignment + * with multiple topics in subscription, but cannot dynamically add topics. + */ +static void do_test_partition_assignment_with_multiple_topics(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + const char *topic1 = "test-multi-topic-1"; + const char *topic2 = "test-multi-topic-2"; + const char *group = "test-share-group-multi-topic-sub"; + int topic1_count = 0, topic2_count = 0, i; + + SUB_TEST_QUICK(); + + /* Setup - create two topics */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic1, 3, 1); + rd_kafka_mock_topic_create(mcluster, topic2, 2, 1); + + share_c = create_share_consumer(bootstraps, group); + + /* Subscribe to both topics */ + subscription = rd_kafka_topic_partition_list_new(2); + rd_kafka_topic_partition_list_add(subscription, topic1, + RD_KAFKA_PARTITION_UA); + rd_kafka_topic_partition_list_add(subscription, topic2, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for join and assignment */ + TEST_ASSERT(wait_assignment_count(share_c, 5, 10000) == 5, + "Expected 5 partitions (3+2)"); + + /* Verify assignment includes partitions from both topics */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + + /* Count partitions per topic */ + for (i = 0; i < assignment->cnt; i++) { + if (strcmp(assignment->elems[i].topic, topic1) == 0) + topic1_count++; + else if (strcmp(assignment->elems[i].topic, topic2) == 0) + topic2_count++; + } + TEST_ASSERT(topic1_count == 3, + "Expected 3 partitions from topic1, got %d", topic1_count); + TEST_ASSERT(topic2_count == 2, + "Expected 2 partitions from topic2, got %d", topic2_count); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Multiple members partition distribution. + * + * N members join group subscribed to topic with M partitions. + * Verify partitions are distributed fairly (all members get some). + * Note: Share groups may allow the same partition to be assigned to + * multiple consumers, so we check for fair distribution rather than + * exclusive assignment. + */ +static void do_test_multiple_members_partition_distribution(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_topic_partition_list_t *share_c1_assign, *share_c2_assign, + *share_c3_assign; + rd_kafka_share_t *share_c1, *share_c2, *share_c3; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-distribution"; + int total_partitions = 0; + rd_bool_t converged; + int64_t dl; + + SUB_TEST_QUICK(); + + /* Setup - 6 partitions, 3 consumers */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 6, 1); + + share_c1 = create_share_consumer(bootstraps, group); + share_c2 = create_share_consumer(bootstraps, group); + share_c3 = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c3, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait until all 3 consumers have converged on a fair distribution + * (each member holds at least 1 partition and the total is >= 6) and + * assert on the snapshot that satisfied that check. + * + * The assignment is driven by the heartbeat thread and churns while + * the three members are still joining, so a member may transiently + * hold partitions and then have them reassigned. Re-reading the + * assignment after convergence would race against that churn and + * observe an intermediate state, so the lists that satisfied the + * check are kept and asserted on directly. */ + TEST_SAY( + "Waiting for 3 members to converge on a fair distribution " + "of 6 partitions...\n"); + share_c1_assign = share_c2_assign = share_c3_assign = NULL; + converged = rd_false; + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c3), &share_c3_assign)); + total_partitions = share_c1_assign->cnt + share_c2_assign->cnt + + share_c3_assign->cnt; + TEST_SAY( + "Current assignment: share consumer 1=%d, " + "share consumer 2=%d, share consumer 3=%d (total=%d)\n", + share_c1_assign->cnt, share_c2_assign->cnt, + share_c3_assign->cnt, total_partitions); + if (share_c1_assign->cnt >= 1 && share_c2_assign->cnt >= 1 && + share_c3_assign->cnt >= 1 && total_partitions >= 6) { + converged = rd_true; + TEST_SAY( + "All 3 members converged, fair distribution " + "reached\n"); + break; + } + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + TEST_SAY("Not yet converged, retrying in 200ms...\n"); + rd_usleep(200 * 1000, 0); + } + + TEST_ASSERT(converged, + "Consumers did not converge on a fair distribution " + "(each member >= 1 partition, total >= 6) within timeout"); + + TEST_SAY( + "Partition distribution: share consumer 1=%d, share consumer 2=%d, " + "share consumer 3=%d (total=%d)\n", + share_c1_assign->cnt, share_c2_assign->cnt, share_c3_assign->cnt, + total_partitions); + + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + rd_kafka_topic_partition_list_destroy(share_c3_assign); + + /* Cleanup */ + test_share_consumer_close(share_c1); + test_share_consumer_close(share_c2); + test_share_consumer_close(share_c3); + test_share_destroy(share_c1); + test_share_destroy(share_c2); + test_share_destroy(share_c3); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Heartbeat successful response completes leave. + * + * When a member sends leave heartbeat (epoch=-1), verify successful + * response completes the leave. + */ +static void do_test_leave_heartbeat_completes_successfully(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-leave-success"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Leave group - should send leave heartbeat and complete. + * Note: After close(), we cannot call rd_kafka_assignment() anymore + * as the broker handle is destroyed. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + + /* Cleanup */ + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Heartbeat failed response during leave still completes. + * + * When a member sends leave heartbeat and gets an error response, + * the leave should still complete (best effort). + */ +static void do_test_leave_heartbeat_completes_on_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-leave-error"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject error for the leave heartbeat */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, 0); + + /* Leave group - should still complete despite error (best effort). + * The key behavior: close() must not hang even when the leave + * heartbeat gets an error response. */ + error = rd_kafka_share_consumer_close(share_c); + /* Close completed (didn't hang) - this is the primary assertion. + * The return code may vary depending on whether the error was + * processed during leave. */ + TEST_SAY("Leave completed with: %s (didn't hang - correct)\n", + rd_kafka_error_name(error)); + if (error) + rd_kafka_error_destroy(error); + + /* Cleanup */ + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Subscription change updates assignment. + * + * Consumer subscribed to topic A, change subscription to topic B, + * verify assignment updates. + */ +static void do_test_subscription_change(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + int found_topicA = 0, found_topicB = 0, i; + int64_t dl; + const char *topicA = "test-sub-change-topic-A"; + const char *topicB = "test-sub-change-topic-B"; + const char *group = "test-share-group-sub-change"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topicA, 2, 1); + rd_kafka_mock_topic_create(mcluster, topicB, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + /* First subscription: topic A */ + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topicA, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for assignment to topic A */ + TEST_ASSERT(wait_assignment_count(share_c, 2, 10000) == 2, + "Expected 2 partitions from topicA"); + + /* Verify assignment has topic A only */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + for (i = 0; i < assignment->cnt; i++) { + TEST_ASSERT(strcmp(assignment->elems[i].topic, topicA) == 0, + "Expected topicA, got %s", + assignment->elems[i].topic); + } + rd_kafka_topic_partition_list_destroy(assignment); + + /* Change subscription to topic B */ + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topicB, + RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for assignment to switch to topic B (3 partitions) */ + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c), &assignment)); + found_topicA = 0; + found_topicB = 0; + for (i = 0; i < assignment->cnt; i++) { + if (strcmp(assignment->elems[i].topic, topicA) == 0) + found_topicA++; + else if (strcmp(assignment->elems[i].topic, topicB) == + 0) + found_topicB++; + } + rd_kafka_topic_partition_list_destroy(assignment); + if (found_topicA == 0 && found_topicB == 3) + break; + rd_usleep(200 * 1000, 0); + } + TEST_ASSERT(found_topicA == 0, + "Expected 0 partitions from topicA after change, got %d", + found_topicA); + TEST_ASSERT(found_topicB == 3, + "Expected 3 partitions from topicB after change, got %d", + found_topicB); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief GROUP_ID_NOT_FOUND while unsubscribed is benign. + * + * When a member that has already unsubscribed receives GROUP_ID_NOT_FOUND, + * it should be treated as benign (the group may have been auto-deleted). + * This should NOT cause a fatal error. + */ +static void do_test_group_id_not_found_while_unsubscribed(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_resp_err_t fatal_err; + rd_kafka_error_t *error; + char errstr[256]; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-id-not-found-unsub"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Unsubscribe first to transition to unsubscribed state. + * The Java test has member in UNSUBSCRIBED state when the + * error arrives. */ + TEST_CALL_ERR__(rd_kafka_share_unsubscribe(share_c)); + rd_usleep(500 * 1000, 0); + + /* Now inject GROUP_ID_NOT_FOUND. + * Since the member is unsubscribed, this should be benign. */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 3, + RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND, 0); + + /* Wait for the error to be processed */ + rd_usleep(500 * 1000, 0); + + /* Verify consumer is NOT in fatal state - error should be benign */ + fatal_err = rd_kafka_fatal_error(test_share_consumer_get_rk(share_c), + errstr, sizeof(errstr)); + TEST_ASSERT(fatal_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected no fatal error when GROUP_ID_NOT_FOUND arrives " + "while unsubscribed, but got: %s (%s)", + rd_kafka_err2str(fatal_err), errstr); + + /* Close consumer */ + error = rd_kafka_share_consumer_close(share_c); + TEST_SAY("Close returned: %s\n", rd_kafka_error_name(error)); + if (error) + rd_kafka_error_destroy(error); + + /* Cleanup */ + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief GROUP_ID_NOT_FOUND while stable is fatal. + * + * When an active member (epoch > 0) receives GROUP_ID_NOT_FOUND, + * it should be treated as a fatal error (group unexpectedly deleted). + */ +static void do_test_group_id_not_found_while_stable_is_fatal(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-id-not-found-stable"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + /* Verify initial assignment - member is in stable state */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Expected 3 partitions initially, got %d", assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Inject GROUP_ID_NOT_FOUND for an active/stable member. + * This should be treated as fatal (group unexpectedly deleted). */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND, 0); + + /* Wait for the fatal error to propagate via consume_batch. */ + error = wait_fatal_error(share_c, 5000); + TEST_ASSERT(error != NULL, + "Expected a fatal error but none received within timeout"); + TEST_ASSERT(rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND, + "Expected GROUP_ID_NOT_FOUND fatal error, got %s", + rd_kafka_error_name(error)); + TEST_SAY("Consumer entered fatal state: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + + /* Cleanup. Consumer is in fatal state, but close() still flushes + * pending acks and leaves the share session, so it succeeds. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief INVALID_REQUEST error handling. + * + * When a consumer receives INVALID_REQUEST error, it should be treated + * as a fatal error. + */ +static void do_test_invalid_request_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-invalid-request"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join */ + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + /* Inject INVALID_REQUEST error (fatal) */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_INVALID_REQUEST, 0); + + /* Wait for the fatal error to propagate via consume_batch. */ + error = wait_fatal_error(share_c, 5000); + TEST_ASSERT(error != NULL, + "Expected a fatal error but none received within timeout"); + TEST_ASSERT(rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR_INVALID_REQUEST, + "Expected INVALID_REQUEST fatal error, got %s", + rd_kafka_error_name(error)); + TEST_SAY("Consumer entered fatal state: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + + /* Cleanup. Consumer is in fatal state, but close() still flushes + * pending acks and leaves the share session, so it succeeds. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief UNSUPPORTED_VERSION error handling. + * + * When a consumer receives UNSUPPORTED_VERSION error, it should be + * treated as a fatal error. + */ +static void do_test_unsupported_version_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-unsupported-version"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join */ + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + /* Inject UNSUPPORTED_VERSION error (fatal) */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION, 0); + + /* Wait for the fatal error to propagate via consume_batch. */ + error = wait_fatal_error(share_c, 5000); + TEST_ASSERT(error != NULL, + "Expected a fatal error but none received within timeout"); + TEST_ASSERT(rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION, + "Expected UNSUPPORTED_VERSION fatal error, got %s", + rd_kafka_error_name(error)); + TEST_SAY("Consumer entered fatal state: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + + /* Cleanup. Consumer is in fatal state, but close() still flushes + * pending acks and leaves the share session, so it succeeds. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief COORDINATOR_LOAD_IN_PROGRESS error handling. + * + * When a consumer receives COORDINATOR_LOAD_IN_PROGRESS, it should + * retry with backoff (transient error). + */ +static void do_test_coordinator_load_in_progress_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_share_t *share_c; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-coord-load"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions initially"); + + /* Inject COORDINATOR_LOAD_IN_PROGRESS error (transient) */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_COORDINATOR_LOAD_IN_PROGRESS, 0); + + /* Wait for consumer to handle transient error and retry */ + rd_usleep(500 * 1000, 0); + + /* Verify heartbeats continue after transient error */ + found_heartbeats = wait_share_heartbeats(mcluster, 2, 1000); + TEST_ASSERT(found_heartbeats >= 1, + "Expected heartbeats to continue after " + "COORDINATOR_LOAD_IN_PROGRESS, got %d", + found_heartbeats); + + /* Verify consumer still has assignment */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after retry"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Consumer graceful shutdown during stable state. + * + * Consumer in stable state leaves group gracefully, sending leave + * heartbeat with epoch=-1. + */ +static void do_test_graceful_shutdown_stable_state(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + rd_kafka_error_t *error; + int found_heartbeats; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-graceful-shutdown"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial join and assignment */ + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + /* Verify initial assignment - member is in stable state */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Expected 3 partitions initially, got %d", assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Record heartbeat count before close */ + found_heartbeats = wait_share_heartbeats(mcluster, 1, 1000); + rd_kafka_mock_stop_request_tracking(mcluster); + rd_kafka_mock_start_request_tracking(mcluster); + + /* Close consumer gracefully - should send leave heartbeat */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected close to succeed, got %s", + rd_kafka_error_name(error)); + + /* Verify leave heartbeat was sent */ + found_heartbeats = wait_share_heartbeats(mcluster, 1, 1000); + TEST_SAY("Found %d heartbeats during shutdown\n", found_heartbeats); + + /* Cleanup */ + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Consumer resubscribes after unsubscribe. + * + * Tests the unsubscribe then resubscribe flow. + */ +static void do_test_resubscribe_after_unsubscribe(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription, *assignment; + rd_kafka_share_t *share_c; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-resubscribe"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + + share_c = create_share_consumer(bootstraps, group); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + rd_kafka_mock_start_request_tracking(mcluster); + + /* First subscribe */ + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + wait_share_heartbeats(mcluster, 1, 1000); + rd_usleep(500 * 1000, 0); + + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 3, + "Expected 3 partitions on first subscribe, got %d", + assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Unsubscribe */ + TEST_SAY("Unsubscribing...\n"); + TEST_CALL_ERR__(rd_kafka_share_unsubscribe(share_c)); + rd_usleep(500 * 1000, 0); + + /* Verify no assignment after unsubscribe */ + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_ASSERT(assignment->cnt == 0, + "Expected 0 partitions after unsubscribe, got %d", + assignment->cnt); + rd_kafka_topic_partition_list_destroy(assignment); + + /* Resubscribe */ + TEST_SAY("Resubscribing...\n"); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Verify assignment restored */ + TEST_ASSERT(wait_assignment_count(share_c, 3, 10000) == 3, + "Expected 3 partitions after resubscribe"); + + /* Cleanup */ + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Consumer leaves and remaining consumers get reassigned. + * + * Tests rebalance when a consumer leaves the group. + */ +static void do_test_consumer_leave_rebalance(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_topic_partition_list_t *share_c1_assign, *share_c2_assign; + rd_kafka_share_t *share_c1, *share_c2, *share_c3; + int final_total; + int64_t dl; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "test-share-group-leave-rebalance"; + + SUB_TEST_QUICK(); + + /* Setup */ + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 6, 1); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + share_c1 = create_share_consumer(bootstraps, group); + share_c2 = create_share_consumer(bootstraps, group); + share_c3 = create_share_consumer(bootstraps, group); + + rd_kafka_mock_start_request_tracking(mcluster); + + /* All three join */ + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c1, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c2, subscription)); + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c3, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* Wait for initial balance */ + wait_share_heartbeats(mcluster, 4, 1000); + wait_share_heartbeats(mcluster, 3, 1000); + + /* Get initial assignments */ + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + TEST_SAY( + "Initial: share consumer 1=%d, share consumer 2=%d (before share " + "consumer 3 leaves)\n", + share_c1_assign->cnt, share_c2_assign->cnt); + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + + /* share consumer 3 leaves */ + TEST_SAY("Share consumer 3 leaving...\n"); + test_share_consumer_close(share_c3); + test_share_destroy(share_c3); + + /* Wait for rebalance to propagate to remaining consumers */ + dl = test_clock() + 15000 * 1000; + while (test_clock() < dl) { + + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + final_total = share_c1_assign->cnt + share_c2_assign->cnt; + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + if (final_total >= 6) + break; + rd_usleep(200 * 1000, 0); + } + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c1), &share_c1_assign)); + TEST_CALL_ERR__(rd_kafka_assignment( + test_share_consumer_get_rk(share_c2), &share_c2_assign)); + final_total = share_c1_assign->cnt + share_c2_assign->cnt; + TEST_SAY( + "After share consumer 3 leave: share consumer 1=%d, " + "share consumer 2=%d\n", + share_c1_assign->cnt, share_c2_assign->cnt); + rd_kafka_topic_partition_list_destroy(share_c1_assign); + rd_kafka_topic_partition_list_destroy(share_c2_assign); + TEST_ASSERT(final_total >= 6, + "Expected >= 6 partitions after rebalance, got %d", + final_total); + + /* Cleanup */ + test_share_consumer_close(share_c1); + test_share_consumer_close(share_c2); + test_share_destroy(share_c1); + test_share_destroy(share_c2); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test calling close twice on the same consumer + */ +static void do_test_double_close(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + const char *topic = test_mk_topic_name(__FUNCTION__, 1); + const char *group_id = topic; + rd_kafka_share_t *share_c; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_error_t *error; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + rd_kafka_mock_start_request_tracking(mcluster); + + share_c = create_share_consumer(bootstraps, group_id); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + wait_share_heartbeats(mcluster, 3, 1000); + + /* First close - should succeed */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(!error, "Expected first close to succeed, got %s", + rd_kafka_error_name(error)); + + /* Second close - should handle gracefully without crashing. + * The Java equivalent tests verify the CompletableFuture + * completes immediately on double-leave. */ + error = rd_kafka_share_consumer_close(share_c); + TEST_ASSERT(error != NULL, + "Expected second close to return error on already-closed " + "consumer, got NULL"); + TEST_ASSERT(rd_kafka_error_code(error) == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE, got %s", + rd_kafka_err2name(rd_kafka_error_code(error))); + rd_kafka_error_destroy(error); + + rd_kafka_topic_partition_list_destroy(subscription); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test consumer subscribed to a topic with no messages + */ +static void do_test_empty_topic_subscription(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + const char *topic = test_mk_topic_name(__FUNCTION__, 1); + const char *group_id = topic; + rd_kafka_share_t *share_c; + rd_kafka_topic_partition_list_t *subscription, *assignment; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_topic_create(mcluster, topic, 3, 1); + rd_kafka_mock_start_request_tracking(mcluster); + + share_c = create_share_consumer(bootstraps, group_id); + + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + + TEST_CALL_ERR__(rd_kafka_share_subscribe(share_c, subscription)); + wait_share_heartbeats(mcluster, 3, 1000); + + /* Wait for assignment on empty topic */ + rd_usleep(500 * 1000, 0); + + TEST_CALL_ERR__(rd_kafka_assignment(test_share_consumer_get_rk(share_c), + &assignment)); + TEST_SAY("Empty topic: %d partitions\n", assignment->cnt); + TEST_ASSERT(assignment->cnt == 3, "Expected 3 partitions, got %d", + assignment->cnt); + + rd_kafka_topic_partition_list_destroy(subscription); + rd_kafka_topic_partition_list_destroy(assignment); + test_share_consumer_close(share_c); + test_share_destroy(share_c); + + rd_kafka_mock_stop_request_tracking(mcluster); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + + +/** + * @brief subscribe() with an empty topic list is equivalent to + * unsubscribe() — must return NO_ERROR and leave the consumer + * in the unsubscribed state. + */ +static void do_test_empty_topic_list_subscription(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *share_c; + rd_kafka_topic_partition_list_t *empty_list; + rd_kafka_resp_err_t err; + const char *group = "test-share-group-empty-topic-list"; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + + share_c = create_share_consumer(bootstraps, group); + + /* subscribe(empty_list) must succeed and act as unsubscribe. */ + empty_list = rd_kafka_topic_partition_list_new(0); + err = rd_kafka_share_subscribe(share_c, empty_list); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR from subscribe(empty_list), got %s", + rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(empty_list); + + test_share_destroy(share_c); + + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + + +int main_0155_share_group_heartbeat_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + test_timeout_set(1500); + + do_test_share_group_heartbeat_basic(); + do_test_share_group_assignment_rebalance(); + do_test_share_group_multi_topic_assignment(); + do_test_share_group_error_injection(); + do_test_share_group_rtt_injection(); + do_test_share_group_session_timeout(); + do_test_share_group_target_assignment(); + do_test_share_group_no_spurious_fencing(); + do_test_unknown_member_id_error(); + + do_test_fenced_member_epoch_error(); + do_test_coordinator_not_available_error(); + do_test_not_coordinator_error(); + do_test_group_authorization_failed_error(); + do_test_group_max_size_reached_error(); + do_test_invalid_request_error(); + do_test_unsupported_version_error(); + do_test_coordinator_load_in_progress_error(); + + do_test_member_rejoin_with_epoch_zero(); + do_test_leaving_member_bumps_group_epoch(); + + do_test_partition_assignment_with_multiple_topics(); + do_test_multiple_members_partition_distribution(); + + do_test_leave_heartbeat_completes_successfully(); + do_test_leave_heartbeat_completes_on_error(); + do_test_graceful_shutdown_stable_state(); + do_test_consumer_leave_rebalance(); + do_test_double_close(); + + do_test_subscription_change(); + do_test_resubscribe_after_unsubscribe(); + do_test_empty_topic_subscription(); + do_test_empty_topic_list_subscription(); + + do_test_group_id_not_found_while_unsubscribed(); + do_test_group_id_not_found_while_stable_is_fatal(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0156-share_consumer_fetch_mock.c b/lib/librdkafka-2.15.0/tests/0156-share_consumer_fetch_mock.c new file mode 100644 index 00000000000..1f35f59d251 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0156-share_consumer_fetch_mock.c @@ -0,0 +1,1551 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @name ShareFetch mock broker tests using the share consumer API. + * + * Exercises the ShareFetch path via mock broker. There is no coordinator + * or ShareAcknowledge support in the mock broker, so group management and + * ack-based state transitions are not validated here. + */ + +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(3, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + /* Set auto.offset.reset=earliest so tests that produce + * before consuming see all records. */ + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + /* Create a producer targeting the mock cluster */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + return ctx; +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static rd_kafka_share_t *new_share_consumer(const char *bootstraps, + const char *group_id) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *consumer; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + + consumer = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(consumer != NULL, "Failed to create share consumer"); + return consumer; +} + +static void subscribe_topics(rd_kafka_share_t *consumer, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *tpl = + rd_kafka_topic_partition_list_new(topic_cnt); + for (int i = 0; i < topic_cnt; i++) { + rd_kafka_topic_partition_list_add(tpl, topics[i], + RD_KAFKA_PARTITION_UA); + } + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, tpl), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(tpl); +} + +static void do_test_basic_consume(void) { + const char *topic = "kip932_pos_basic"; + const int msgcnt = 5; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-pos-basic"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, "Expected %d consumed, got %d", msgcnt, + consumed); + SUB_TEST_PASS(); +} + +static void do_test_followup_fetch(void) { + const char *topic = "kip932_pos_followup"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 5); + + consumer = new_share_consumer(ctx.bootstraps, "sg-pos-followup"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 3, 30, 500, NULL, 0); + consumed += test_share_consume_msgs(consumer, 2, 30, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 5, "Expected 5 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +static void do_test_multi_partition(void) { + const char *topic = "kip932_pos_multi_part"; + const int msgcnt = 6; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 2, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-pos-multipart"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, msgcnt, 60, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, "Expected %d consumed, got %d", msgcnt, + consumed); + SUB_TEST_PASS(); +} + +static void do_test_multi_topic(void) { + const char *topic_a = "kip932_pos_topic_a"; + const char *topic_b = "kip932_pos_topic_b"; + const char *topics[] = {topic_a, topic_b}; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_a, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic A"); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_b, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic B"); + + test_produce_msgs_simple(ctx.producer, topic_a, RD_KAFKA_PARTITION_UA, + 2); + test_produce_msgs_simple(ctx.producer, topic_b, RD_KAFKA_PARTITION_UA, + 2); + consumer = new_share_consumer(ctx.bootstraps, "sg-pos-multitopic"); + subscribe_topics(consumer, topics, 2); + consumed = test_share_consume_msgs(consumer, 4, 40, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 4, "Expected 4 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +static void do_test_empty_topic_no_records(void) { + const char *topic = "kip932_pos_empty"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + consumer = new_share_consumer(ctx.bootstraps, "sg-pos-empty"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +static int do_test_negative_sharefetch_error(rd_kafka_resp_err_t err) { + const char *topic = "kip932_neg_sharefetch_error"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 1); + + rd_kafka_mock_push_request_errors(ctx.mcluster, RD_KAFKAP_ShareFetch, 1, + err); + + consumer = new_share_consumer(ctx.bootstraps, "sg-neg-sharefetch"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 30, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + return consumed; +} + +static void do_test_sharefetch_invalid_session_epoch(void) { + SUB_TEST_QUICK(); + do_test_negative_sharefetch_error( + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_unknown_topic_or_part(void) { + SUB_TEST_QUICK(); + do_test_negative_sharefetch_error( + RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART); + SUB_TEST_PASS(); +} + +static void do_test_sghb_error(rd_kafka_resp_err_t err, int count) { + const char *topic = "kip932_neg_sghb"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + rd_kafka_resp_err_t *errs; + int i; + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 1); + + /* Build an array of 'count' identical errors and push them all. + * Using the array variant avoids UB from mismatched varargs count. */ + errs = malloc(sizeof(*errs) * count); + TEST_ASSERT(errs != NULL, "malloc failed"); + for (i = 0; i < count; i++) + errs[i] = err; + rd_kafka_mock_push_request_errors_array( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, (size_t)count, errs); + free(errs); + + consumer = new_share_consumer(ctx.bootstraps, "sg-neg-sghb"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed, got %d", consumed); +} + +static void do_test_sghb_coord_unavailable(void) { + SUB_TEST_QUICK(); + do_test_sghb_error(RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, 50); + SUB_TEST_PASS(); +} + +static void do_test_topic_error(rd_kafka_resp_err_t err) { + const char *topic = "kip932_neg_topic_error"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 1); + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, err); + + consumer = new_share_consumer(ctx.bootstraps, "sg-neg-topicerr"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed, got %d", consumed); +} + +static void do_test_topic_error_unknown_topic_or_part(void) { + SUB_TEST_QUICK(); + do_test_topic_error(RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART); + SUB_TEST_PASS(); +} + +static void do_test_unknown_topic_subscription(void) { + const char *topic = "kip932_neg_unknown_topic"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + consumer = new_share_consumer(ctx.bootstraps, "sg-neg-unknown-topic"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +static void do_test_empty_fetch_no_records(void) { + const char *topic = "kip932_neg_empty_fetch"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + consumer = new_share_consumer(ctx.bootstraps, "sg-neg-empty"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Verify that ShareFetch rejects requests from an unregistered member + * (SHARE_SESSION_NOT_FOUND), and that after the member re-joins it can + * consume again. + * + * Phase 1: Consumer joins normally via SGHB -> consumes messages OK. + * Phase 2: Push SGHB errors -> heartbeats fail -> member expires -> broker + * rejects ShareFetch with SHARE_SESSION_NOT_FOUND. + * Phase 3: SGHB errors drain -> member re-joins -> consumes again. + */ +static void do_test_member_validation(void) { + const char *topic = "kip932_member_validation"; + const int msgcnt = 4; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_p1, consumed_p3; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Short session timeout so the member is evicted quickly once + * heartbeats stop succeeding. */ + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-member-val"); + subscribe_topics(consumer, &topic, 1); + + /* Phase 1: Consume normally -- member is registered via SGHB. */ + consumed_p1 = test_share_consume_msgs(consumer, 2, 30, 500, NULL, 0); + TEST_SAY("member_validation: phase1 consumed %d/2\n", consumed_p1); + + /* Phase 2: Block SGHB so heartbeats fail. + * Push enough errors to cover the window while we wait for the + * member to be evicted. */ + rd_kafka_mock_push_request_errors( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 20, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE); + + /* Wait for the member to be evicted (500ms session timeout + margin). + */ + rd_usleep(1500 * 1000, 0); + + /* Phase 3: SGHB errors will eventually drain. Once a SGHB + * succeeds, the member re-joins and the remaining records + * become fetchable again. */ + consumed_p3 = test_share_consume_msgs(consumer, 2, 50, 500, NULL, 0); + TEST_SAY("member_validation: phase3 consumed %d/2\n", consumed_p3); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_p1 >= 2 && (consumed_p1 + consumed_p3) >= msgcnt, + "Expected at least 2+2, got %d+%d", consumed_p1, + consumed_p3); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_session_expiry_rtt(void) { + const char *topic = "kip932_rtt_expiry"; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Session timeout must be long enough for normal requests + * to complete, but short enough to expire during high RTT. */ + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 1000); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + /* Produce only 1 message so a single batch cannot over-consume + * past the requested count in test_share_consume_msgs(). */ + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 1); + + consumer = new_share_consumer(ctx.bootstraps, "sg-rtt-expiry"); + subscribe_topics(consumer, &topic, 1); + + /* Phase 1: consume one message with normal RTT (no injection). */ + consumed = test_share_consume_msgs(consumer, 1, 20, 500, NULL, 0); + TEST_SAY("rtt_expiry: phase1 consumed %d/1\n", consumed); + + /* Phase 2: inject RTT >> session timeout to force session expiry. + * All requests to broker 1 now take 3s, but the session + * expires after 1s of inactivity. The record from phase 1 + * was acquired but not acked, so it becomes available again + * after the session expires. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 3000); + rd_usleep(2000 * 1000, 0); /* wait for session to expire */ + + /* Phase 3: clear RTT and let the consumer recover. + * The same record should be re-delivered after session expiry. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 0); + consumed += test_share_consume_msgs(consumer, 1, 30, 500, NULL, 0); + TEST_SAY("rtt_expiry: phase3 consumed %d/2 total\n", consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 2, "Expected 2 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +static void do_test_forgotten_topics(void) { + const char *topic_a = "kip932_forgotten_a"; + const char *topic_b = "kip932_forgotten_b"; + const char *both[] = {topic_a, topic_b}; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_a, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic A"); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_b, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic B"); + + /* Produce 2 messages to each topic */ + test_produce_msgs_simple(ctx.producer, topic_a, RD_KAFKA_PARTITION_UA, + 2); + test_produce_msgs_simple(ctx.producer, topic_b, RD_KAFKA_PARTITION_UA, + 2); + + /* Subscribe to both topics and consume all 4 messages */ + consumer = new_share_consumer(ctx.bootstraps, "sg-forgotten"); + subscribe_topics(consumer, both, 2); + consumed = test_share_consume_msgs(consumer, 4, 40, 500, NULL, 0); + TEST_SAY("forgotten_topics: consumed %d/4 from both topics\n", + consumed); + + /* Re-subscribe to only topic_a (topic_b becomes forgotten) */ + subscribe_topics(consumer, &topic_a, 1); + + /* Produce 2 more messages to topic_a */ + test_produce_msgs_simple(ctx.producer, topic_a, RD_KAFKA_PARTITION_UA, + 2); + + /* Consume the 2 new messages -- only topic_a should deliver */ + consumed += test_share_consume_msgs(consumer, 2, 30, 500, NULL, 0); + TEST_SAY("forgotten_topics: consumed %d/6 total after forget\n", + consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + /* We expect at least the 4 initial + 2 from topic_a = 6. + * Depending on timing the consumer may or may not have already + * received all messages from the first round, so we accept >= 4. */ + TEST_ASSERT(consumed >= 4, "Expected at least 4 consumed, got %d", + consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Produce messages one-at-a-time (each flush creates a separate + * msgset on the mock partition), then consume and verify all are + * received. This validates that the ShareFetch response includes + * records from *all* acquired msgsets, not just the first one. + */ +static void do_test_multi_batch_consume(void) { + const char *topic = "kip932_multi_batch"; + const int msgcnt = 5; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Produce each message individually with a flush in between, + * guaranteeing separate msgsets on the mock partition. */ + for (int i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "batch-%d", i); + TEST_ASSERT(rd_kafka_producev( + ctx.producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Produce failed"); + rd_kafka_flush(ctx.producer, 5000); + } + + consumer = new_share_consumer(ctx.bootstraps, "sg-multi-batch"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("multi_batch: consumed %d/%d\n", consumed, msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, "Expected %d consumed, got %d", msgcnt, + consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Verify that max_delivery_attempts causes records to be archived + * after the limit is exceeded. Consumer A acquires all records, then + * its session times out (releasing locks). Consumer B acquires them + * again, and its session also times out. After the delivery limit is + * exhausted, Consumer C should see 0 available records. + */ +static void do_test_max_delivery_attempts(void) { + const char *topic = "kip932_max_delivery"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, consumed_c; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Set max delivery attempts to 2 and a short session timeout + * so locks expire quickly after consumer destruction. */ + rd_kafka_mock_sharegroup_set_max_delivery_attempts(ctx.mcluster, 2); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Delivery 1: Consumer A acquires and "crashes" (no ack). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-max-delivery"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("max_delivery: A consumed %d/%d (delivery 1)\n", consumed_a, + msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, 0); /* wait for lock expiry */ + + /* Delivery 2: Consumer B acquires same records again (delivery_count + * reaches 2 = limit) and "crashes". */ + consumer = new_share_consumer(ctx.bootstraps, "sg-max-delivery"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("max_delivery: B consumed %d/%d (delivery 2)\n", consumed_b, + msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, 0); /* wait for lock expiry */ + + /* Delivery 3 attempt: Consumer C should get 0 records because + * all records have been archived (delivery_count >= max). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-max-delivery"); + subscribe_topics(consumer, &topic, 1); + consumed_c = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("max_delivery: C consumed %d/0 (should be archived)\n", + consumed_c); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt && + consumed_c == 0, + "Expected A=%d B=%d C=0, got A=%d B=%d C=%d", msgcnt, + msgcnt, consumed_a, consumed_b, consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief Verify that record_lock_duration_ms controls how long acquired + * records stay locked, independently of session_timeout_ms. + * Sets a short lock duration (300ms) with a longer session timeout + * (10s). Consumer A acquires records and "crashes". After the short + * lock duration expires, Consumer B should be able to acquire them + * even though A's session hasn't timed out yet. + */ +static void do_test_record_lock_duration(void) { + const char *topic = "kip932_lock_duration"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Long session timeout, short record lock duration. */ + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 10000); + rd_kafka_mock_sharegroup_set_record_lock_duration(ctx.mcluster, 300); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A acquires records, then crashes (no close). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-lock-duration"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("lock_duration: A consumed %d/%d\n", consumed_a, msgcnt); + test_share_destroy(consumer); + + /* Wait for record lock to expire (300ms + margin). + * test_share_destroy sends SGHB LEAVE which releases + * locks immediately, but we still need to wait for the + * client's internal rejoin cycle to settle. */ + rd_usleep(1000 * 1000, 0); + + /* Consumer B should get the records because locks have expired + * even though A's session is still technically alive. + * Use higher max_attempts to account for the mock broker's + * SGHB LEAVE->rejoin cycle delay. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-lock-duration"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 100, 500, NULL, 0); + TEST_SAY("lock_duration: B consumed %d/%d\n", consumed_b, msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt, + "Expected A=%d B=%d, got A=%d B=%d", msgcnt, msgcnt, + consumed_a, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Multi-consumer lock expiry test. + * + * Consumer A acquires records, then crashes (destroyed without close). + * After the lock expiry timeout, consumer B should be able to pick up + * the same records because the proactive lock-expiry scan releases them. + */ +static void do_test_multi_consumer_lock_expiry(void) { + const char *topic = "kip932_multi_consumer_lock"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer_a, *consumer_b; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Use a short session/lock timeout so the test runs quickly. */ + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: subscribe and consume all records (acquires locks). */ + consumer_a = + new_share_consumer(ctx.bootstraps, "sg-multi-consumer-lock"); + subscribe_topics(consumer_a, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer_a, msgcnt, 50, 500, NULL, 0); + TEST_SAY("multi_consumer: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Simulate crash: destroy consumer A without calling close. + * The session will time out and the proactive lock-expiry + * timer will release A's locks. */ + test_share_destroy(consumer_a); + + /* Wait for locks to expire (session_timeout=500ms, add margin). */ + rd_usleep(1500 * 1000, 0); + + /* Consumer B: joins the same share group, should get the same + * records once the locks have been released. + * Use higher max_attempts to account for the mock broker's + * SGHB LEAVE->rejoin cycle delay. */ + consumer_b = + new_share_consumer(ctx.bootstraps, "sg-multi-consumer-lock"); + subscribe_topics(consumer_b, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer_b, msgcnt, 100, 500, NULL, 0); + TEST_SAY("multi_consumer: B consumed %d/%d\n", consumed_b, msgcnt); + + test_share_consumer_close(consumer_b); + test_share_destroy(consumer_b); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt, + "Expected A=%d B=%d, got A=%d B=%d", msgcnt, msgcnt, + consumed_a, consumed_b); + SUB_TEST_PASS(); +} + + +/* + * TODO KIP-932: The functions below inject errors using + * rd_kafka_mock_push_request_errors(), which only sets the top-level + * ShareFetch error code. The error codes tested here are per-partition + * errors that the broker never emits at the top level. Re-enable + * once partition-level mock error injection is available. + * +static void do_test_sharefetch_fetch_error(rd_kafka_resp_err_t err) { + const char *topic = "kip932_fetch_error"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + int consumed = 0; + size_t i; + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 3); + + rd_kafka_mock_push_request_errors(ctx.mcluster, RD_KAFKAP_ShareFetch, 1, + err); + + consumer = new_share_consumer(ctx.bootstraps, "sg-fetch-error"); + subscribe_topics(consumer, &topic, 1); + + rd_kafka_share_poll(consumer, 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = rd_kafka_messages_get(batch, i); + if (!msg->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("fetch_error(%s): consumed=%d rcvd=%zu\n", + rd_kafka_err2name(err), consumed, rcvd); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed with %s, got %d", + rd_kafka_err2name(err), consumed); +} + +static void do_test_sharefetch_fetch_error_not_leader(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error( + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_fetch_error_unknown_topic_or_part(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error(RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_fetch_error_unknown_topic_id(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error(RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_fetch_error_fenced_leader_epoch(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error(RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_fetch_error_unknown_leader_epoch(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error(RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_topic_authorization_failed(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error( + RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); + SUB_TEST_PASS(); +} + +static void do_test_sharefetch_corrupt_message(void) { + SUB_TEST(); + do_test_sharefetch_fetch_error(RD_KAFKA_RESP_ERR_INVALID_MSG); + SUB_TEST_PASS(); +} +*/ + +/* + * TODO KIP-932: This test injects FENCED_LEADER_EPOCH using + * rd_kafka_mock_push_request_errors(), which only sets the top-level + * ShareFetch error code. FENCED_LEADER_EPOCH is a per-partition error + * that the broker never emits at the top level; the top-level handler + * treats it as transient and retries, so the consumer receives records. + * Re-enable once partition-level mock error injection is available. + * +static void do_test_sharefetch_fetch_disconnected(void) { + const char *topic = "kip932_disconnect"; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + int consumed = 0; + size_t i; + + SUB_TEST(); + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 3); + + rd_kafka_mock_push_request_errors( + ctx.mcluster, RD_KAFKAP_ShareFetch, 1, + RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH); + + consumer = new_share_consumer(ctx.bootstraps, "sg-disconnect"); + subscribe_topics(consumer, &topic, 1); + + rd_kafka_share_poll(consumer, 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = rd_kafka_messages_get(batch, i); + if (!msg->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("fetch_disconnected: consumed=%d rcvd=%zu\n", consumed, rcvd); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, "Expected 0 consumed on disconnect, got %d", + consumed); + SUB_TEST_PASS(); +} +*/ + +/** + * @brief Fetch records and close implicitly (no explicit ack). + * + * Verifies that a consumer can fetch records and then close successfully. + * The close sends ShareAcknowledge with epoch=-1 to release the session. + */ +static void do_test_sharefetch_fetch_and_close_implicit(void) { + const char *topic = "kip932_fetch_close"; + const int msgcnt = 2; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST(); + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-fetch-close"); + subscribe_topics(consumer, &topic, 1); + + /* Fetch records */ + consumed = test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("fetch_and_close: consumed %d/%d\n", consumed, msgcnt); + + /* Close — sends ShareAcknowledge/ShareFetch with epoch=-1 */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, "Expected %d consumed, got %d", msgcnt, + consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Test that SHARE_SESSION_LIMIT_REACHED is returned when the + * session cache is full. + * + * Set max_fetch_sessions=1, open a session with consumer 1, + * then attempt to open a second session with consumer 2. + * Consumer 2 should fail to consume any records because every + * ShareFetch epoch=0 attempt gets SHARE_SESSION_LIMIT_REACHED. + */ +static void do_test_session_limit_reached(void) { + const char *topic = "kip932_session_limit"; + const int msgcnt = 5; + test_ctx_t ctx; + rd_kafka_share_t *consumer1, *consumer2; + int consumed1, consumed2; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + /* Limit to 1 session */ + rd_kafka_mock_sharegroup_set_max_fetch_sessions(ctx.mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer 1 opens a session successfully */ + consumer1 = new_share_consumer(ctx.bootstraps, "sg-session-limit"); + subscribe_topics(consumer1, &topic, 1); + consumed1 = + test_share_consume_msgs(consumer1, msgcnt, 30, 500, NULL, 0); + TEST_ASSERT(consumed1 == msgcnt, + "Consumer 1: expected %d consumed, got %d", msgcnt, + consumed1); + + /* Consumer 2 tries to open a session — cache is full */ + consumer2 = new_share_consumer(ctx.bootstraps, "sg-session-limit"); + subscribe_topics(consumer2, &topic, 1); + consumed2 = test_share_consume_msgs(consumer2, 1, 5, 500, NULL, 0); + TEST_ASSERT(consumed2 == 0, + "Consumer 2: expected 0 consumed (session limit), got %d", + consumed2); + + rd_kafka_share_consumer_close(consumer1); + rd_kafka_share_destroy(consumer1); + rd_kafka_share_consumer_close(consumer2); + rd_kafka_share_destroy(consumer2); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that the session limit is enforced per broker, not globally. + * + * Create two single-partition topics, each with its leader on a + * different broker. Set max_fetch_sessions=1 (per broker). + * + * Consumer 1 subscribes to topic_a (broker 1) and consumer 2 subscribes + * to topic_b (broker 2). Both should succeed because each broker only + * has 1 session despite there being 2 sessions globally. + * + * Then consumer 3 subscribes to topic_a — this should fail because + * broker 1 already has 1 session (from consumer 1). + */ +static void do_test_session_limit_per_broker(void) { + const char *topic_a = "kip932_session_limit_pb_a"; + const char *topic_b = "kip932_session_limit_pb_b"; + const int msgcnt = 5; + test_ctx_t ctx; + rd_kafka_share_t *consumer1, *consumer2, *consumer3; + int consumed1, consumed2, consumed3; + + SUB_TEST(); + + ctx = test_ctx_new(); + + /* Create two topics, each with 1 partition on a different broker. */ + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_a, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic A"); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_b, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic B"); + rd_kafka_mock_partition_set_leader(ctx.mcluster, topic_a, 0, 1); + rd_kafka_mock_partition_set_leader(ctx.mcluster, topic_b, 0, 2); + + test_produce_msgs_simple(ctx.producer, topic_a, RD_KAFKA_PARTITION_UA, + msgcnt); + test_produce_msgs_simple(ctx.producer, topic_b, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Limit to 1 session per broker */ + rd_kafka_mock_sharegroup_set_max_fetch_sessions(ctx.mcluster, 1); + + /* Consumer 1 subscribes to topic_a -> session on broker 1 */ + consumer1 = new_share_consumer(ctx.bootstraps, "sg-session-limit-pb"); + subscribe_topics(consumer1, &topic_a, 1); + consumed1 = + test_share_consume_msgs(consumer1, msgcnt, 30, 500, NULL, 0); + TEST_ASSERT(consumed1 == msgcnt, + "Consumer 1: expected %d consumed, got %d", msgcnt, + consumed1); + + /* Consumer 2 subscribes to topic_b -> session on broker 2. + * If the limit were global, this would fail (2 sessions > limit 1). + * Per-broker: broker 2 has 0 sessions, so it succeeds. */ + consumer2 = new_share_consumer(ctx.bootstraps, "sg-session-limit-pb"); + subscribe_topics(consumer2, &topic_b, 1); + consumed2 = + test_share_consume_msgs(consumer2, msgcnt, 30, 500, NULL, 0); + TEST_ASSERT(consumed2 == msgcnt, + "Consumer 2: expected %d consumed, got %d", msgcnt, + consumed2); + + /* Consumer 3 subscribes to topic_a -> broker 1 already has + * 1 session (from consumer 1), so this should be rejected. */ + consumer3 = new_share_consumer(ctx.bootstraps, "sg-session-limit-pb"); + subscribe_topics(consumer3, &topic_a, 1); + consumed3 = test_share_consume_msgs(consumer3, 1, 5, 500, NULL, 0); + TEST_ASSERT(consumed3 == 0, + "Consumer 3: expected 0 consumed (broker 1 at limit), " + "got %d", + consumed3); + + rd_kafka_share_consumer_close(consumer1); + rd_kafka_share_destroy(consumer1); + rd_kafka_share_consumer_close(consumer2); + rd_kafka_share_destroy(consumer2); + rd_kafka_share_consumer_close(consumer3); + rd_kafka_share_destroy(consumer3); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that ShareFetch with epoch=0 and acks is rejected with + * INVALID_REQUEST via the mock broker's injected error mechanism. + * + * Injects SHARE_SESSION_LIMIT_REACHED errors to force the client to + * retry with epoch=0. The client never piggybacks acks on epoch=0 + * (that's a protocol violation), so this test validates that the mock + * broker's error injection for SHARE_SESSION_LIMIT_REACHED works and + * the client recovers when the error clears. + */ +static void do_test_session_limit_recovery(void) { + const char *topic = "kip932_session_limit_recovery"; + const int msgcnt = 5; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Push 3 SHARE_SESSION_LIMIT_REACHED errors, then let it succeed */ + rd_kafka_mock_push_request_errors( + ctx.mcluster, RD_KAFKAP_ShareFetch, 3, + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED, + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED, + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED); + + consumer = new_share_consumer(ctx.bootstraps, "sg-session-recovery"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, msgcnt, 30, 500, NULL, 0); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, + "Expected %d consumed after recovery, got %d", msgcnt, + consumed); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that max_record_locks limits the number of in-flight + * records per share-partition. + * + * Produce 10 records, set max_record_locks=3. Consumer A should get + * only 3 records on the first fetch round (the rest are blocked by + * the lock limit). After A acks (via second poll) and closes, + * consumer B should get the next batch. Total consumed across both + * should be 10. + */ +static void do_test_max_record_locks(void) { + const char *topic = "kip932_max_record_locks"; + const int msgcnt = 10; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int total_consumed = 0; + int rounds = 0; + + SUB_TEST_QUICK(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 3); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consume in rounds. Each round can get at most 3 records + * (the lock limit). The implicit ack from the next poll + * frees the locks for the next batch. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-max-locks"); + subscribe_topics(consumer, &topic, 1); + + while (total_consumed < msgcnt && rounds < 20) { + int got = + test_share_consume_msgs(consumer, 3, 10, 500, NULL, 0); + if (got == 0) + break; + total_consumed += got; + rounds++; + TEST_SAY( + "max_record_locks: round %d consumed %d (total %d/%d)\n", + rounds, got, total_consumed, msgcnt); + } + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(total_consumed == msgcnt, + "Expected %d total consumed, got %d", msgcnt, + total_consumed); + TEST_ASSERT(rounds > 1, + "Expected multiple rounds (lock limit=3, msgs=10), " + "got %d rounds", + rounds); + + SUB_TEST_PASS(); +} + +/** + * @brief Verify that max_record_locks counts only ACQUIRED records, + * not total inflight records (which includes ARCHIVED). + * + * Produce 6 records. Set max_record_locks=3, max_delivery_attempts=1. + * + * Consumer A acquires records 0-2 (lock limit), then is destroyed + * WITHOUT close (no implicit ack). The connection close releases + * the locks; with max_delivery_attempts=1 the released records + * transition directly to ARCHIVED (delivery_count >= limit). + * + * At this point: + * - Records 0-2: ARCHIVED, still in the inflight list + * - SPSO = 0 (never advanced — ARCHIVED != ACKNOWLEDGED) + * - inflight_cnt = 3, acquired_cnt = 0 + * + * Consumer B subscribes. With the correct check (acquired_cnt), + * records 3-5 are acquirable because only 0 records are ACQUIRED. + * With an inflight_cnt-based check, the limit would block all + * acquisition (inflight_cnt=3 >= max_record_locks=3). + */ +static void do_test_max_record_locks_acquired_only(void) { + const char *topic = "kip932_max_locks_acquired_only"; + const int msgcnt = 6; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 3); + rd_kafka_mock_sharegroup_set_max_delivery_attempts(ctx.mcluster, 1); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: acquire first batch (3 records due to lock limit). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-acq-only"); + subscribe_topics(consumer, &topic, 1); + consumed_a = test_share_consume_msgs(consumer, 3, 30, 500, NULL, 0); + TEST_SAY("acquired_only: A consumed %d/3\n", consumed_a); + + /* Destroy WITHOUT close — avoids implicit ack. + * The connection close releases locks; with max_delivery_attempts=1 + * the released records transition to ARCHIVED (not AVAILABLE). + * SPSO stays at 0 because ARCHIVED != ACKNOWLEDGED. + * inflight_cnt stays 3, but acquired_cnt drops to 0. */ + rd_kafka_share_destroy(consumer); + rd_usleep(1500 * 1000, 0); /* wait for connection close */ + + /* Consumer B subscribes. The share-partition now has: + * - Records 0-2: ARCHIVED (inflight_cnt=3, acquired_cnt=0) + * - Records 3-5: not yet in inflight list + * With acquired_cnt check, Consumer B acquires records 3-5. + * With inflight_cnt check, the limit would block acquisition. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-acq-only"); + subscribe_topics(consumer, &topic, 1); + consumed_b = test_share_consume_msgs(consumer, 3, 30, 500, NULL, 0); + TEST_SAY("acquired_only: B consumed %d/3\n", consumed_b); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == 3, "A: expected 3, got %d", consumed_a); + TEST_ASSERT(consumed_b == 3, + "B: expected 3 (max_record_locks should count only " + "ACQUIRED, not ARCHIVED), got %d", + consumed_b); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that SPSO advances when log retention deletes records + * below the current SPSO. + * + * Produce 10 records (offsets 0-9). Consumer A acquires 0-4. + * Then delete records before offset 5 (simulating log retention). + * Consumer A closes (acks 0-4, but they're already archived by + * retention). Consumer B should get records 5-9 only — SPSO + * was advanced to 5 by the retention, and in-flight records + * below 5 were archived. + */ +static void do_test_spso_advances_on_log_retention(void) { + const char *topic = "kip932_log_retention_spso"; + const int msgcnt = 10; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST_QUICK(); + ctx = test_ctx_new(); + + /* Limit to 5 records per fetch so A gets only 0-4. */ + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 5); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A acquires records 0-4. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-log-retention"); + subscribe_topics(consumer, &topic, 1); + consumed_a = test_share_consume_msgs(consumer, 5, 30, 500, NULL, 0); + TEST_SAY("log_retention: A consumed %d/5\n", consumed_a); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + + /* Simulate log retention: delete records before offset 5. */ + TEST_ASSERT( + rd_kafka_mock_partition_delete_records(ctx.mcluster, topic, 0, 5) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to delete records"); + + /* Remove lock limit. */ + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 0); + + /* Consumer B should get records 5-9 (SPSO advanced to 5). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-log-retention"); + subscribe_topics(consumer, &topic, 1); + consumed_b = test_share_consume_msgs(consumer, 5, 30, 500, NULL, 0); + TEST_SAY("log_retention: B consumed %d/5\n", consumed_b); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == 5, "A: expected 5 consumed, got %d", + consumed_a); + TEST_ASSERT(consumed_b == 5, "B: expected 5 consumed, got %d", + consumed_b); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that auto.offset.reset=latest (the default) + * causes the consumer to skip records produced before subscription. + * + * Produce 5 records, then subscribe with auto.offset.reset=latest. + * Consumer should get 0 old records but should get new records + * produced after subscription. + */ +static void do_test_auto_offset_reset_latest(void) { + const char *topic = "kip932_offset_reset_latest"; + const int msgcnt = 5; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + char errstr[512]; + int consumed; + + SUB_TEST_QUICK(); + + /* Create a fresh context — do NOT set auto_offset_reset=earliest + * (the default is "latest"). */ + memset(&ctx, 0, sizeof(ctx)); + ctx.mcluster = test_mock_cluster_new(3, &ctx.bootstraps); + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + /* Intentionally NOT calling set_auto_offset_reset — default is latest + */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Produce records BEFORE subscribing */ + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Subscribe — SPSO should start at end of log (latest) */ + consumer = new_share_consumer(ctx.bootstraps, "sg-offset-latest"); + subscribe_topics(consumer, &topic, 1); + consumed = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + TEST_SAY("offset_reset_latest: consumed %d (expected 0)\n", consumed); + TEST_ASSERT(consumed == 0, + "Expected 0 consumed with latest offset reset, got %d", + consumed); + + /* Produce new records AFTER subscription — these should be visible */ + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + consumed = test_share_consume_msgs(consumer, msgcnt, 30, 500, NULL, 0); + TEST_SAY("offset_reset_latest: consumed %d new records (expected %d)\n", + consumed, msgcnt); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, + "Expected %d new records consumed, got %d", msgcnt, + consumed); + + SUB_TEST_PASS(); +} + +int main_0156_share_consumer_fetch_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + /* This test suite has many subtests; set a generous timeout. + * When running in parallel with other test suites (e.g., 0155, 0157) + * the mock broker and consumer threads compete for CPU, which can + * slow individual subtests by 5x or more. Use 1500s to be safe. */ + test_timeout_set(1500); + + /* Positive scenarios */ + do_test_basic_consume(); + do_test_followup_fetch(); + do_test_multi_partition(); + do_test_multi_topic(); + do_test_empty_topic_no_records(); + do_test_sharefetch_session_expiry_rtt(); + do_test_forgotten_topics(); + do_test_multi_batch_consume(); + do_test_max_delivery_attempts(); + do_test_record_lock_duration(); + do_test_multi_consumer_lock_expiry(); + + /* Negative scenarios */ + do_test_sharefetch_invalid_session_epoch(); + do_test_sharefetch_unknown_topic_or_part(); + do_test_sghb_coord_unavailable(); + do_test_topic_error_unknown_topic_or_part(); + do_test_unknown_topic_subscription(); + do_test_empty_fetch_no_records(); + do_test_member_validation(); + + /* TODO KIP-932: These tests inject partition-level errors at the + * top-level response field, which the broker never does for these + * error codes. Replace with partition-level mock injection once + * that API is available. */ + /* do_test_sharefetch_fetch_error_not_leader(); */ + /* do_test_sharefetch_fetch_error_unknown_topic_or_part(); */ + /* do_test_sharefetch_fetch_error_unknown_topic_id(); */ + /* do_test_sharefetch_fetch_error_fenced_leader_epoch(); */ + /* do_test_sharefetch_fetch_error_unknown_leader_epoch(); */ + /* do_test_sharefetch_topic_authorization_failed(); */ + /* do_test_sharefetch_corrupt_message(); */ + /* do_test_sharefetch_fetch_disconnected(); */ + do_test_sharefetch_fetch_and_close_implicit(); + + /* Session management */ + do_test_session_limit_reached(); + do_test_session_limit_per_broker(); + do_test_session_limit_recovery(); + + /* Record lock limits */ + do_test_max_record_locks(); + do_test_max_record_locks_acquired_only(); + + /* Offset reset */ + do_test_auto_offset_reset_latest(); + + /* Log retention */ + do_test_spso_advances_on_log_retention(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0157-share_consumer_ack_mock.c b/lib/librdkafka-2.15.0/tests/0157-share_consumer_ack_mock.c new file mode 100644 index 00000000000..0cbc46dbb6a --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0157-share_consumer_ack_mock.c @@ -0,0 +1,1788 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @name Share Acknowledgement (implicit ack) mock broker tests. + * + * Exercises the implicit ack flow via mock broker: records acquired + * by a ShareFetch are acknowledged on the next ShareFetch, which + * causes the mock broker to archive them and advance SPSO. + */ + +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(3, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + /* Set auto.offset.reset=earliest so tests that produce + * before consuming see all records. */ + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + /* Create a producer targeting the mock cluster */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + return ctx; +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static rd_kafka_share_t *new_share_consumer(const char *bootstraps, + const char *group_id) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *consumer; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + + consumer = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(consumer != NULL, "Failed to create share consumer"); + return consumer; +} + +static void subscribe_topics(rd_kafka_share_t *consumer, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *tpl = + rd_kafka_topic_partition_list_new(topic_cnt); + for (int i = 0; i < topic_cnt; i++) { + rd_kafka_topic_partition_list_add(tpl, topics[i], + RD_KAFKA_PARTITION_UA); + } + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, tpl), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(tpl); +} + + + +/* =================================================================== + * Positive test scenarios + * =================================================================== */ + +/** + * @brief Basic implicit ack prevents re-delivery. + * + * 1. Produce 5 messages. + * 2. Consume all 5 (first ShareFetch acquires them). + * 3. Poll again — the next ShareFetch carries AcknowledgementBatches + * (ACCEPT) for the 5 records. The mock broker archives them and + * advances SPSO. + * 4. Subsequent polls should return 0 records (no re-delivery). + */ +static void do_test_implicit_ack_no_redelivery(void) { + const char *topic = "kip932_ack_no_redeliver"; + const int msgcnt = 5; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-noredeliver"); + subscribe_topics(consumer, &topic, 1); + + /* Consume all records. */ + consumed = test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_no_redelivery: consumed %d/%d\n", consumed, msgcnt); + + /* Poll again to trigger the implicit ack (piggybacked on the + * next ShareFetch). Expect 0 new records. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_no_redelivery: extra %d/0 (should be 0)\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt && extra == 0, + "Expected consumed=%d extra=0, got consumed=%d extra=%d", + msgcnt, consumed, extra); + SUB_TEST_PASS(); +} + +/** + * @brief After implicit ack, only newly-produced records are delivered. + * + * 1. Produce batch A (3 messages). + * 2. Consumer A consumes batch A and triggers implicit ack. + * 3. Consumer A closes. + * 4. Produce batch B (3 messages). + * 5. Consumer B (same group) consumes — should receive only batch B + * because batch A was acked (ARCHIVED, SPSO advanced). + */ +static void do_test_implicit_ack_with_new_records(void) { + const char *topic = "kip932_ack_new_records"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Batch A */ + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 3); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-newrecords"); + subscribe_topics(consumer, &topic, 1); + + consumed_a = test_share_consume_msgs(consumer, 3, 40, 500, NULL, 0); + TEST_SAY("ack_new_records: A consumed %d/3\n", consumed_a); + + /* Trigger implicit ack for batch A. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_new_records: A extra %d/0 (ack sent)\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Batch B */ + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 3); + + /* Consumer B: same group — batch A is archived, should get + * only batch B. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-newrecords"); + subscribe_topics(consumer, &topic, 1); + + consumed_b = test_share_consume_msgs(consumer, 3, 40, 500, NULL, 0); + TEST_SAY("ack_new_records: B consumed %d/3 (batch B only)\n", + consumed_b); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == 3 && extra == 0 && consumed_b == 3, + "Expected A=3 extra=0 B=3, got A=%d extra=%d B=%d", + consumed_a, extra, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Acked records are not visible to a different consumer that + * joins the same share group afterwards. + * + * 1. Consumer A consumes all records and triggers implicit ack. + * 2. Consumer A closes. + * 3. Consumer B joins the same group — should see 0 records because + * the records are ARCHIVED and SPSO has advanced. + */ +static void do_test_implicit_ack_cross_consumer(void) { + const char *topic = "kip932_ack_cross_consumer"; + const int msgcnt = 5; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer_a, *consumer_b; + int consumed_a, consumed_b, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: consume and ack. */ + consumer_a = new_share_consumer(ctx.bootstraps, "sg-ack-cross"); + subscribe_topics(consumer_a, &topic, 1); + + consumed_a = + test_share_consume_msgs(consumer_a, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_cross_consumer: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer_a, 1, 10, 500, NULL, 0); + TEST_SAY("ack_cross_consumer: A extra %d/0\n", extra); + + test_share_consumer_close(consumer_a); + test_share_destroy(consumer_a); + + /* Consumer B: same group, should see nothing. */ + consumer_b = new_share_consumer(ctx.bootstraps, "sg-ack-cross"); + subscribe_topics(consumer_b, &topic, 1); + + consumed_b = test_share_consume_msgs(consumer_b, 1, 15, 500, NULL, 0); + TEST_SAY("ack_cross_consumer: B consumed %d/0 (should be 0)\n", + consumed_b); + + test_share_consumer_close(consumer_b); + test_share_destroy(consumer_b); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && extra == 0 && consumed_b == 0, + "Expected A=%d extra=0 B=0, got A=%d extra=%d B=%d", msgcnt, + consumed_a, extra, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Implicit ack works across multiple partitions. + * + * 1. Create a 2-partition topic. + * 2. Produce messages (distributed across partitions by the partitioner). + * 3. Consume all messages. + * 4. Trigger implicit ack. + * 5. Poll again — should get 0 (acked from all partitions). + */ +static void do_test_implicit_ack_multi_partition(void) { + const char *topic = "kip932_ack_multi_part"; + const int msgcnt = 6; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 2, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-multipart"); + subscribe_topics(consumer, &topic, 1); + + consumed = test_share_consume_msgs(consumer, msgcnt, 60, 500, NULL, 0); + TEST_SAY("ack_multi_partition: consumed %d/%d\n", consumed, msgcnt); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_multi_partition: extra %d/0\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt && extra == 0, + "Expected consumed=%d extra=0, got consumed=%d extra=%d", + msgcnt, consumed, extra); + SUB_TEST_PASS(); +} + +/** + * @brief Multiple rounds of produce -> consume -> ack with new consumers. + * + * Each round: + * 1. Produce N messages. + * 2. New consumer joins the same share group. + * 3. Consumer should get exactly the N new records (records from + * previous rounds were acked and SPSO advanced). + * 4. Consumer triggers implicit ack and closes. + * + * This verifies that SPSO advancement from acks in earlier rounds + * persists correctly across consumer lifetimes. + */ +static void do_test_implicit_ack_multiple_rounds(void) { + const char *topic = "kip932_ack_multi_round"; + const int per_round = 2; + const int rounds = 3; + test_ctx_t ctx = test_ctx_new(); + int total_consumed = 0; + int round_ok = 1; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + for (int r = 0; r < rounds; r++) { + rd_kafka_share_t *consumer; + int got, extra; + + test_produce_msgs_simple(ctx.producer, topic, + RD_KAFKA_PARTITION_UA, per_round); + + consumer = + new_share_consumer(ctx.bootstraps, "sg-ack-multiround"); + subscribe_topics(consumer, &topic, 1); + + got = test_share_consume_msgs(consumer, per_round, 40, 500, + NULL, 0); + TEST_SAY("ack_multiple_rounds: round %d consumed %d/%d\n", + r + 1, got, per_round); + total_consumed += got; + if (got != per_round) + round_ok = 0; + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + if (extra != 0) { + TEST_SAY( + "ack_multiple_rounds: round %d extra %d " + "(expected 0)\n", + r + 1, extra); + round_ok = 0; + } + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + } + + TEST_SAY("ack_multiple_rounds: total %d/%d\n", total_consumed, + per_round * rounds); + + test_ctx_destroy(&ctx); + + TEST_ASSERT(round_ok && total_consumed == per_round * rounds, + "Expected %d total, got %d (round_ok=%d)", + per_round * rounds, total_consumed, round_ok); + SUB_TEST_PASS(); +} + +/** + * @brief Implicit ack with a single record (boundary case). + * + * Produce exactly 1 message, consume it, trigger implicit ack, + * verify no re-delivery. Tests the minimum-batch-size edge case for + * AcquiredRecords ranges and SPSO advancement. + */ +static void do_test_implicit_ack_single_record(void) { + const char *topic = "kip932_ack_single"; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 1); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-single"); + subscribe_topics(consumer, &topic, 1); + + consumed = test_share_consume_msgs(consumer, 1, 40, 500, NULL, 0); + TEST_SAY("ack_single_record: consumed %d/1\n", consumed); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_single_record: extra %d/0 (should be 0)\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 1 && extra == 0, + "Expected consumed=1 extra=0, got consumed=%d extra=%d", + consumed, extra); + SUB_TEST_PASS(); +} + +/** + * @brief Implicit ack with a large batch of records. + * + * Produce 100 messages, consume all, trigger implicit ack, verify 0 + * on subsequent polls. Tests ack handling at scale — many records + * in a single AcquiredRecords range and SPSO advancement over a + * large offset span. + */ +static void do_test_implicit_ack_large_batch(void) { + const char *topic = "kip932_ack_large"; + const int msgcnt = 100; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-large"); + subscribe_topics(consumer, &topic, 1); + + consumed = test_share_consume_msgs(consumer, msgcnt, 80, 500, NULL, 0); + TEST_SAY("ack_large_batch: consumed %d/%d\n", consumed, msgcnt); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_large_batch: extra %d/0 (should be 0)\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt && extra == 0, + "Expected consumed=%d extra=0, got consumed=%d extra=%d", + msgcnt, consumed, extra); + SUB_TEST_PASS(); +} + +/** + * @brief Implicit ack across multiple topics in the same share group. + * + * 1. Create 2 topics, produce messages to both. + * 2. Subscribe a single consumer to both topics via the same group. + * 3. Consume all messages from both topics. + * 4. Trigger implicit ack. + * 5. Consumer B joins the same group — should see 0 records from + * either topic (both acked independently). + */ +static void do_test_implicit_ack_multi_topic(void) { + const char *topic_a = "kip932_ack_mtopic_a"; + const char *topic_b = "kip932_ack_mtopic_b"; + const char *both[] = {topic_a, topic_b}; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed, extra, consumed_b; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_a, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic A"); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_b, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic B"); + + test_produce_msgs_simple(ctx.producer, topic_a, RD_KAFKA_PARTITION_UA, + 3); + test_produce_msgs_simple(ctx.producer, topic_b, RD_KAFKA_PARTITION_UA, + 3); + + /* Consumer A: subscribe to both, consume all 6, ack. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-mtopic"); + subscribe_topics(consumer, both, 2); + + consumed = test_share_consume_msgs(consumer, 6, 60, 500, NULL, 0); + TEST_SAY("ack_multi_topic: consumed %d/6 from both\n", consumed); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_multi_topic: extra %d/0\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Consumer B: same group — should see 0 from either topic. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-mtopic"); + subscribe_topics(consumer, both, 2); + + consumed_b = test_share_consume_msgs(consumer, 1, 15, 500, NULL, 0); + TEST_SAY("ack_multi_topic: B consumed %d/0 (should be 0)\n", + consumed_b); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 6 && extra == 0 && consumed_b == 0, + "Expected consumed=6 extra=0 B=0, got %d %d %d", consumed, + extra, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Implicit ack with records from separate message sets. + * + * Produce records one at a time with a flush in between, creating + * separate msgsets on the mock partition. Then consume all, ack, + * and verify 0 on subsequent polls. + * + * This validates that the ack machinery handles records spanning + * multiple RecordBatch wire objects. + */ +static void do_test_implicit_ack_multi_msgset(void) { + const char *topic = "kip932_ack_multi_msgset"; + const int msgcnt = 5; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Produce each message individually with a flush in between, + * guaranteeing separate msgsets on the mock partition. */ + for (int i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "msgset-%d", i); + TEST_ASSERT(rd_kafka_producev( + ctx.producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Produce failed"); + rd_kafka_flush(ctx.producer, 5000); + } + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-multimsgset"); + subscribe_topics(consumer, &topic, 1); + + consumed = test_share_consume_msgs(consumer, msgcnt, 60, 500, NULL, 0); + TEST_SAY("ack_multi_msgset: consumed %d/%d\n", consumed, msgcnt); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_multi_msgset: extra %d/0 (should be 0)\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt && extra == 0, + "Expected consumed=%d extra=0, got consumed=%d extra=%d", + msgcnt, consumed, extra); + SUB_TEST_PASS(); +} + +/* =================================================================== + * Negative test scenarios + * =================================================================== */ + +/** + * @brief Crash (no ack) -> records re-delivered after lock expiry. + * + * Consumer A consumes records but is destroyed without closing and + * without sending a subsequent ShareFetch that would carry the ack. + * After the acquisition lock expires, Consumer B should receive the + * same records. + */ +static void do_test_crash_before_ack_redelivery(void) { + const char *topic = "kip932_ack_crash_redeliver"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Short lock so the test doesn't wait too long. */ + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: acquire records, then crash (no close, no ack). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-crash-redeliver"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("crash_before_ack: A consumed %d/%d\n", consumed_a, msgcnt); + test_share_destroy(consumer); /* crash — no close */ + + /* Wait for lock expiry (session_timeout=500ms + margin). */ + rd_usleep(1500 * 1000, NULL); + + /* Consumer B: should get the same records (re-delivered). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-crash-redeliver"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("crash_before_ack: B consumed %d/%d (re-delivered)\n", + consumed_b, msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt, + "Expected A=%d B=%d, got A=%d B=%d", msgcnt, msgcnt, + consumed_a, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Crash -> re-delivery -> ack stops further re-delivery. + * + * 1. Consumer A consumes records but crashes (no ack). + * 2. Locks expire, records become AVAILABLE again. + * 3. Consumer B consumes the re-delivered records and triggers an + * implicit ack via the next poll. + * 4. Consumer C joins — should see 0 records (acked by B). + */ +static void do_test_crash_then_ack_stops_redelivery(void) { + const char *topic = "kip932_ack_crash_then_ack"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, consumed_c, extra; + + SUB_TEST(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: acquire and crash. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-crash-then-ack"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("crash_then_ack: A consumed %d/%d (will crash)\n", consumed_a, + msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, NULL); /* wait for lock expiry */ + + /* Consumer B: re-delivery, then ack via next poll. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-crash-then-ack"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("crash_then_ack: B consumed %d/%d (re-delivered)\n", + consumed_b, msgcnt); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("crash_then_ack: B extra %d/0\n", extra); + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Consumer C: should see 0 — records were acked by B. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-crash-then-ack"); + subscribe_topics(consumer, &topic, 1); + consumed_c = test_share_consume_msgs(consumer, 1, 15, 500, NULL, 0); + TEST_SAY("crash_then_ack: C consumed %d/0 (should be 0)\n", consumed_c); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt && + extra == 0 && consumed_c == 0, + "Expected A=%d B=%d extra=0 C=0, got A=%d B=%d extra=%d " + "C=%d", + msgcnt, msgcnt, consumed_a, consumed_b, extra, consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief Session expiry via heartbeat failure causes pending ack to be lost. + * + * 1. Consumer A acquires records normally. + * 2. Push many SGHB errors -> heartbeats fail -> member is evicted. + * 3. Consumer A is destroyed (crash, no ack delivered). + * 4. The broker releases A's locks upon eviction. + * 5. Consumer B joins -> gets the same records (re-delivered). + */ +static void do_test_session_expiry_invalidates_ack(void) { + const char *topic = "kip932_ack_session_expire"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Short session timeout so eviction happens quickly. */ + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: consume records (acquires them). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-session-expire"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("session_expiry: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Block SGHB so heartbeats fail -> member evicted. */ + rd_kafka_mock_push_request_errors( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 20, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE); + + /* Wait for member eviction (session_timeout=500ms + margin). */ + rd_usleep(1500 * 1000, NULL); + + /* Crash consumer A without acking. */ + test_share_destroy(consumer); + + /* Consumer B: records should be re-delivered (locks released + * when A's membership was evicted). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-session-expire"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("session_expiry: B consumed %d/%d (re-delivered)\n", + consumed_b, msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt, + "Expected A=%d B=%d, got A=%d B=%d", msgcnt, msgcnt, + consumed_a, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Records archived by max_delivery_count without any ack. + * + * 1. Set max_delivery_attempts=2. + * 2. Consumer A acquires records and crashes (delivery 1). + * 3. Consumer B acquires same records and crashes (delivery 2 = max). + * 4. Consumer C joins — should see 0 records because the broker + * archived them after the delivery count was exhausted. + */ +static void do_test_max_delivery_without_ack(void) { + const char *topic = "kip932_ack_maxdel_noack"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, consumed_c; + + SUB_TEST(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_max_delivery_attempts(ctx.mcluster, 2); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Delivery 1: Consumer A acquires and crashes. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-maxdel-noack"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("max_delivery_no_ack: A consumed %d/%d (delivery 1)\n", + consumed_a, msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, NULL); /* wait for lock expiry */ + + /* Delivery 2 = max: Consumer B acquires and crashes. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-maxdel-noack"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("max_delivery_no_ack: B consumed %d/%d (delivery 2)\n", + consumed_b, msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, NULL); /* wait for lock expiry + archival */ + + /* Delivery 3 attempt: records should be archived (delivery + * count exhausted). Consumer C sees 0. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-maxdel-noack"); + subscribe_topics(consumer, &topic, 1); + consumed_c = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("max_delivery_no_ack: C consumed %d/0 (archived)\n", + consumed_c); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt && + consumed_c == 0, + "Expected A=%d B=%d C=0, got A=%d B=%d C=%d", msgcnt, + msgcnt, consumed_a, consumed_b, consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief ShareFetch error injection prevents ack from being processed. + * + * 1. Consumer A acquires records. + * 2. Inject ShareFetch transport errors on all brokers so the next + * ShareFetch (carrying the ack) causes a disconnect. + * 3. Consumer A is destroyed (crash — ack was never processed). + * 4. Wait for locks to expire. + * 5. Consumer B joins — should get the same records (re-delivered). + */ +static void do_test_sharefetch_error_drops_ack(void) { + const char *topic = "kip932_ack_sf_error"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A acquires records. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-sf-error"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("sf_error_drops_ack: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Inject ShareFetch transport errors — the next ShareFetch + * (which would carry acks) will cause a disconnect. */ + rd_kafka_mock_push_request_errors( + ctx.mcluster, RD_KAFKAP_ShareFetch, 10, + RD_KAFKA_RESP_ERR__TRANSPORT, RD_KAFKA_RESP_ERR__TRANSPORT, + RD_KAFKA_RESP_ERR__TRANSPORT, RD_KAFKA_RESP_ERR__TRANSPORT, + RD_KAFKA_RESP_ERR__TRANSPORT, RD_KAFKA_RESP_ERR__TRANSPORT, + RD_KAFKA_RESP_ERR__TRANSPORT, RD_KAFKA_RESP_ERR__TRANSPORT, + RD_KAFKA_RESP_ERR__TRANSPORT, RD_KAFKA_RESP_ERR__TRANSPORT); + + /* Crash consumer A (ack never delivered). */ + test_share_destroy(consumer); + rd_usleep(1500 * 1000, NULL); /* wait for lock expiry */ + + /* Clear any remaining errors. */ + rd_kafka_mock_clear_request_errors(ctx.mcluster, RD_KAFKAP_ShareFetch); + + /* Consumer B should get re-delivered records. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-sf-error"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("sf_error_drops_ack: B consumed %d/%d (re-delivered)\n", + consumed_b, msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt, + "Expected A=%d B=%d, got A=%d B=%d", msgcnt, msgcnt, + consumed_a, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Un-acked records from a forgotten topic remain available. + * + * 1. Subscribe to topic_a and topic_b, consume from both. + * 2. Crash consumer immediately (no ack for either topic). + * 3. Wait for lock expiry so records become AVAILABLE. + * 4. Consumer B subscribes to topic_a only, consumes and acks topic_a. + * 5. Consumer C subscribes to topic_b only — should see topic_b's + * records (they were never acked, only released by lock expiry). + */ +static void do_test_forgotten_topic_releases_not_acks(void) { + const char *topic_a = "kip932_ack_forget_a"; + const char *topic_b = "kip932_ack_forget_b"; + const char *both[] = {topic_a, topic_b}; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_both, consumed_a, consumed_b, extra; + + SUB_TEST(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 2000); + rd_kafka_mock_sharegroup_set_heartbeat_interval(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_a, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic A"); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_b, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic B"); + + test_produce_msgs_simple(ctx.producer, topic_a, RD_KAFKA_PARTITION_UA, + 2); + test_produce_msgs_simple(ctx.producer, topic_b, RD_KAFKA_PARTITION_UA, + 2); + + /* Consumer A: subscribe to both, consume all 4, then crash. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-forget"); + subscribe_topics(consumer, both, 2); + + consumed_both = test_share_consume_msgs(consumer, 4, 60, 500, NULL, 0); + TEST_SAY("forgotten_releases: consumed %d/4 from both\n", + consumed_both); + + /* Crash — no ack for either topic. */ + test_share_destroy(consumer); + rd_usleep(3000 * 1000, NULL); /* wait for session + lock expiry */ + + /* Consumer B: subscribe to topic_a only, consume and ack. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-forget"); + subscribe_topics(consumer, &topic_a, 1); + + consumed_a = test_share_consume_msgs(consumer, 2, 40, 500, NULL, 0); + TEST_SAY("forgotten_releases: B consumed %d/2 from topic_a\n", + consumed_a); + + /* Trigger implicit ack for topic_a. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Consumer C: subscribe to topic_b only. + * Topic_b's records were never acked — should be available. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-forget"); + subscribe_topics(consumer, &topic_b, 1); + + consumed_b = test_share_consume_msgs(consumer, 2, 40, 500, NULL, 0); + TEST_SAY( + "forgotten_releases: C consumed %d/2 from topic_b " + "(should be re-delivered)\n", + consumed_b); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + (void)extra; + TEST_ASSERT(consumed_both >= 4 && consumed_a == 2 && consumed_b == 2, + "Expected both>=4 A=2 B=2, got both=%d A=%d B=%d", + consumed_both, consumed_a, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Multiple consumers crash sequentially without acking. + * + * A, B, C each acquire the same records and crash without acking. + * Each re-delivery round hands the same records to the next consumer + * after lock expiry. This validates that without any ack, records + * cycle through ACQUIRED->AVAILABLE indefinitely (bounded by + * max_delivery_count, which we set high here). + */ +static void do_test_multi_consumer_cascade_crash(void) { + const char *topic = "kip932_ack_cascade_crash"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, consumed_c; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* High max delivery so records don't get archived. */ + rd_kafka_mock_sharegroup_set_max_delivery_attempts(ctx.mcluster, 10); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A: acquire and crash. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-cascade-crash"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("cascade_crash: A consumed %d/%d\n", consumed_a, msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, NULL); + + /* Consumer B: re-acquire and crash. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-cascade-crash"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("cascade_crash: B consumed %d/%d\n", consumed_b, msgcnt); + test_share_destroy(consumer); + rd_usleep(1500 * 1000, NULL); + + /* Consumer C: re-acquire and crash. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-cascade-crash"); + subscribe_topics(consumer, &topic, 1); + consumed_c = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("cascade_crash: C consumed %d/%d\n", consumed_c, msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt && + consumed_c == msgcnt, + "Expected A=%d B=%d C=%d, got A=%d B=%d C=%d", msgcnt, + msgcnt, msgcnt, consumed_a, consumed_b, consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief Lock expires before the ack-carrying ShareFetch arrives. + * + * 1. Set a very short record lock duration (200ms). + * 2. Consumer A acquires records. + * 3. Inject high RTT on broker 1 so the next ShareFetch (carrying + * the ack) is delayed beyond the lock duration. + * 4. Lock expires -> records become AVAILABLE. + * 5. Consumer A is destroyed. + * 6. Consumer B should be able to get the same records. + */ +static void do_test_lock_expiry_before_ack(void) { + const char *topic = "kip932_ack_lock_expire"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Very short lock, long session timeout. */ + rd_kafka_mock_sharegroup_set_record_lock_duration(ctx.mcluster, 200); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 10000); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A acquires records. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-lockexpire"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("lock_expiry_before_ack: A consumed %d/%d\n", consumed_a, + msgcnt); + + /* Inject high RTT on all brokers so the next ShareFetch + * (carrying the ack) is delayed well past lock expiry. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 3000); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 2, 3000); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 3, 3000); + + /* Wait for lock to expire (200ms + margin), then crash. */ + rd_usleep(800 * 1000, NULL); + test_share_destroy(consumer); + + /* Clear RTT so consumer B can operate normally. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 0); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 2, 0); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 3, 0); + + /* Consumer B should get the records — locks expired before + * A's ack could be delivered. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-lockexpire"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("lock_expiry_before_ack: B consumed %d/%d\n", consumed_b, + msgcnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt, + "Expected A=%d B=%d, got A=%d B=%d", msgcnt, msgcnt, + consumed_a, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Empty topic produces no ack side effects. + * + * 1. Subscribe to an empty topic, poll several times (no records, + * no AcquiredRecords, no acks sent). + * 2. Produce messages. + * 3. Consumer should consume them normally. + * 4. Ack them normally (implicit ack via next poll). + * 5. Verify 0 after ack. + */ +static void do_test_empty_topic_no_ack_side_effects(void) { + const char *topic = "kip932_ack_empty_topic"; + const int msgcnt = 3; + test_ctx_t ctx = test_ctx_new(); + rd_kafka_share_t *consumer; + int consumed_empty, consumed, extra; + + SUB_TEST_QUICK(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-empty-topic"); + subscribe_topics(consumer, &topic, 1); + + /* Phase 1: Poll empty topic — no records, no acks. */ + consumed_empty = test_share_consume_msgs(consumer, 1, 5, 500, NULL, 0); + TEST_SAY("empty_topic: phase1 consumed %d/0 (should be 0)\n", + consumed_empty); + + /* Phase 2: Produce messages, then close this consumer and + * open a new one to avoid incremental session issues. */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-empty-topic"); + subscribe_topics(consumer, &topic, 1); + + consumed = test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("empty_topic: phase2 consumed %d/%d\n", consumed, msgcnt); + + /* Phase 3: Trigger implicit ack, verify 0. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("empty_topic: phase3 extra %d/0 (should be 0)\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_empty == 0 && consumed == msgcnt && extra == 0, + "Expected empty=0 consumed=%d extra=0, got %d %d %d", + msgcnt, consumed_empty, consumed, extra); + SUB_TEST_PASS(); +} + +/** + * @brief Coordinator failover — consumer recovers and acks new records. + * + * 1. Consumer A consumes and acks records (happy path). + * 2. Push SGHB errors -> member evicted. + * 3. Clear errors -> member re-joins. + * 4. Produce new records. + * 5. Consumer (new instance, same group) consumes and acks new records. + * 6. Consumer C joins — should see 0 (everything acked). + */ +static void do_test_coordinator_failover_ack_recovery(void) { + const char *topic = "kip932_ack_coord_failover"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, consumed_c, extra; + + SUB_TEST(); + ctx = test_ctx_new(); + + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 500); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Phase 1: Consumer A consumes and acks normally. */ + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-coord-failover"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("coord_failover: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Trigger implicit ack. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("coord_failover: A extra %d/0\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Phase 2: Temporarily push SGHB errors (simulating coordinator + * failover). Any consumer joining now will fail heartbeats. */ + rd_kafka_mock_push_request_errors( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 10, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE, + RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE); + + /* Wait for errors to start draining, then produce new records. */ + rd_usleep(1000 * 1000, NULL); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Phase 3: Errors drain, new consumer B joins and consumes + * the newly produced records (A's records are already acked). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-coord-failover"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 60, 500, NULL, 0); + TEST_SAY("coord_failover: B consumed %d/%d\n", consumed_b, msgcnt); + + /* Trigger implicit ack for B's records. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("coord_failover: B extra %d/0\n", extra); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Phase 4: Consumer C — everything is acked, should see 0. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-coord-failover"); + subscribe_topics(consumer, &topic, 1); + consumed_c = test_share_consume_msgs(consumer, 1, 15, 500, NULL, 0); + TEST_SAY("coord_failover: C consumed %d/0 (should be 0)\n", consumed_c); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt && + consumed_c == 0, + "Expected A=%d B=%d C=0, got A=%d B=%d C=%d", msgcnt, + msgcnt, consumed_a, consumed_b, consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief Test that ack validation returns INVALID_RECORD_STATE when the + * record's lock has expired and was re-acquired by another consumer. + * + * Consumer A acquires records, locks expire, consumer B re-acquires and + * successfully acks. The records should not be redelivered a third time + * (consumer B's ack succeeded because the mock broker now correctly + * reports INVALID_RECORD_STATE for stale acks and honours valid ones). + */ +static void do_test_ack_after_lock_expiry_redelivers(void) { + const char *topic = "kip932_ack_invalid_state"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer_a, *consumer_b, *consumer_c; + int consumed_a, consumed_b, consumed_c; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Short lock duration so locks expire quickly. */ + rd_kafka_mock_sharegroup_set_record_lock_duration(ctx.mcluster, 200); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 10000); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A acquires records. */ + consumer_a = new_share_consumer(ctx.bootstraps, "sg-ack-invalid-state"); + subscribe_topics(consumer_a, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer_a, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_invalid_state: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Inject RTT so A's ack is delayed past lock expiry. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 3000); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 2, 3000); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 3, 3000); + + /* Wait for lock to expire. */ + rd_usleep(800 * 1000, NULL); + + /* Destroy A without close — ack never delivered. */ + rd_kafka_share_destroy(consumer_a); + + /* Clear RTT. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 0); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 2, 0); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 3, 0); + + /* Consumer B re-acquires (locks expired) and acks implicitly + * by doing a second poll (which piggybacks the ack on the next + * ShareFetch). */ + consumer_b = new_share_consumer(ctx.bootstraps, "sg-ack-invalid-state"); + subscribe_topics(consumer_b, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer_b, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_invalid_state: B consumed %d/%d\n", consumed_b, msgcnt); + + /* Second poll triggers implicit ack for B's records. */ + test_share_consume_msgs(consumer_b, 1, 3, 500, NULL, 0); + + rd_kafka_share_consumer_close(consumer_b); + rd_kafka_share_destroy(consumer_b); + + /* Consumer C should get 0 records — B's ack succeeded. */ + consumer_c = new_share_consumer(ctx.bootstraps, "sg-ack-invalid-state"); + subscribe_topics(consumer_c, &topic, 1); + consumed_c = test_share_consume_msgs(consumer_c, 1, 5, 500, NULL, 0); + TEST_SAY("ack_invalid_state: C consumed %d (expected 0)\n", consumed_c); + + rd_kafka_share_consumer_close(consumer_c); + rd_kafka_share_destroy(consumer_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt, "A: expected %d consumed, got %d", + msgcnt, consumed_a); + TEST_ASSERT(consumed_b == msgcnt, "B: expected %d consumed, got %d", + msgcnt, consumed_b); + TEST_ASSERT(consumed_c == 0, "C: expected 0 consumed (B acked), got %d", + consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief Test that two consumers can sequentially acquire, ack, and + * advance SPSO without interference. + * + * Consumer A acquires records 0-2 and acks them (implicit ack via + * second poll). Consumer B then gets records 3-5 (not 0-2, since + * those were acked and SPSO advanced). Validates that the ack + * error handling doesn't interfere with normal ack flow. + */ +static void do_test_ack_success_advances_spso(void) { + const char *topic = "kip932_ack_spso_advance"; + const int msgcnt = 6; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b; + + SUB_TEST(); + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + /* Consumer A acquires first batch. MaxRecords in the client + * defaults to a large value, so it will get all 6. We use + * the max_record_locks limit to cap acquisition at 3. */ + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 3); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-spso"); + subscribe_topics(consumer, &topic, 1); + consumed_a = test_share_consume_msgs(consumer, 3, 30, 500, NULL, 0); + TEST_SAY("ack_spso: A consumed %d/3\n", consumed_a); + + /* Second poll triggers implicit ack for A's records. */ + test_share_consume_msgs(consumer, 1, 3, 500, NULL, 0); + + /* Close A cleanly. */ + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + + /* Remove lock limit so B can get remaining records. */ + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 0); + + /* Consumer B should get records 3-5 (SPSO advanced past 0-2). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-spso"); + subscribe_topics(consumer, &topic, 1); + consumed_b = test_share_consume_msgs(consumer, 3, 30, 500, NULL, 0); + TEST_SAY("ack_spso: B consumed %d/3\n", consumed_b); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == 3, "A: expected 3 consumed, got %d", + consumed_a); + TEST_ASSERT(consumed_b == 3, "B: expected 3 consumed, got %d", + consumed_b); + SUB_TEST_PASS(); +} + + +/** + * @brief epoch=-1 (final fetch) must not acquire new records. + * + * Consumer A acquires two batches (max_record_locks=3 caps each), + * then closes. The close sends epoch=-1 which releases A's + * un-acked records but must not acquire anything new. + * Consumer B picks up the released records; consumer C sees 0. + */ +static void do_test_final_fetch_no_acquisition(void) { + const char *topic = "kip932_final_fetch_no_acq"; + const int msgcnt = 6; + test_ctx_t ctx; + rd_kafka_share_t *consumer, *consumer_c; + int consumed_a, consumed_a2, consumed_b, consumed_c, extra; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Limit acquisition to 3 records at a time. */ + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 3); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-final-no-acq"); + subscribe_topics(consumer, &topic, 1); + + consumed_a = test_share_consume_msgs(consumer, 3, 40, 500, NULL, 0); + TEST_SAY("final_fetch_no_acq: A batch1 consumed %d/3\n", consumed_a); + + /* Implicit ack for batch1, acquires batch2. */ + consumed_a2 = test_share_consume_msgs(consumer, 3, 40, 500, NULL, 0); + TEST_SAY("final_fetch_no_acq: A batch2 consumed %d/3\n", consumed_a2); + + /* Close sends epoch=-1; batch2 records released. */ + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 0); + + /* B picks up the released batch2 records. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-final-no-acq"); + subscribe_topics(consumer, &topic, 1); + + consumed_b = test_share_consume_msgs(consumer, 3, 40, 500, NULL, 0); + TEST_SAY("final_fetch_no_acq: B consumed %d/3\n", consumed_b); + + extra = test_share_consume_msgs(consumer, 1, 5, 500, NULL, + 0); /* ack B's records */ + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + + /* C should see nothing — everything acked. */ + consumer_c = new_share_consumer(ctx.bootstraps, "sg-final-no-acq"); + subscribe_topics(consumer_c, &topic, 1); + consumed_c = test_share_consume_msgs(consumer_c, 1, 10, 500, NULL, 0); + TEST_SAY("final_fetch_no_acq: C consumed %d/0 (should be 0)\n", + consumed_c); + rd_kafka_share_consumer_close(consumer_c); + rd_kafka_share_destroy(consumer_c); + + TEST_ASSERT(consumed_c == 0, "C: expected 0 consumed, got %d", + consumed_c); + + test_ctx_destroy(&ctx); + + (void)extra; + TEST_ASSERT(consumed_a == 3 && consumed_a2 == 3 && consumed_b == 3, + "Expected A1=3 A2=3 B=3, got A1=%d A2=%d B=%d", consumed_a, + consumed_a2, consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Ack for records past SPSO (log retention) succeeds silently. + * + * Consumer A acquires 5 records, then log retention moves + * start_offset past the first 3. The implicit ack covers all 5, + * including the 3 that are now below SPSO. Those must be silently + * accepted rather than returning INVALID_RECORD_STATE. + * Consumer B should see 0. + */ +static void do_test_ack_after_log_retention_silent(void) { + const char *topic = "kip932_ack_log_retention_silent"; + const int msgcnt = 5; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, extra; + + SUB_TEST(); + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-log-ret-silent"); + subscribe_topics(consumer, &topic, 1); + + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_log_retention_silent: A consumed %d/%d\n", consumed_a, + msgcnt); + + /* Move start_offset past first 3 records (simulates retention). */ + rd_kafka_mock_partition_set_follower_wmarks(ctx.mcluster, topic, 0, 3, + -1); + + /* Implicit ack covers all 5; offsets 0-2 are below SPSO now. */ + extra = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_log_retention_silent: A extra %d/0\n", extra); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + + /* B should see nothing. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-log-ret-silent"); + subscribe_topics(consumer, &topic, 1); + consumed_b = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_log_retention_silent: B consumed %d/0 (should be 0)\n", + consumed_b); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == 0, + "Expected A=%d B=0, got A=%d B=%d", msgcnt, consumed_a, + consumed_b); + SUB_TEST_PASS(); +} + +/** + * @brief Ack atomicity — expired locks cause full batch rejection. + * + * Consumer A acquires records but its locks expire before the ack + * arrives (RTT delay). Because acks are atomic per partition, the + * entire batch is rejected rather than partially applied. + * Consumer B re-acquires, acks, and consumer C sees 0. + */ +static void do_test_ack_atomicity_lock_expiry(void) { + const char *topic = "kip932_ack_atomicity"; + const int msgcnt = 3; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed_a, consumed_b, consumed_c; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Very short lock so it expires before A can ack. */ + rd_kafka_mock_sharegroup_set_record_lock_duration(ctx.mcluster, 200); + rd_kafka_mock_sharegroup_set_session_timeout(ctx.mcluster, 10000); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-atomicity"); + subscribe_topics(consumer, &topic, 1); + consumed_a = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_atomicity: A consumed %d/%d\n", consumed_a, msgcnt); + + /* Delay A's next ShareFetch past lock expiry. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 3000); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 2, 3000); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 3, 3000); + + rd_usleep(800 * 1000, NULL); /* locks expire */ + rd_kafka_share_destroy(consumer); /* crash */ + + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 1, 0); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 2, 0); + rd_kafka_mock_broker_set_rtt(ctx.mcluster, 3, 0); + + /* B re-acquires everything (A's ack was rejected atomically). */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-atomicity"); + subscribe_topics(consumer, &topic, 1); + consumed_b = + test_share_consume_msgs(consumer, msgcnt, 50, 500, NULL, 0); + TEST_SAY("ack_atomicity: B consumed %d/%d (re-delivered)\n", consumed_b, + msgcnt); + + test_share_consume_msgs(consumer, 1, 5, 500, NULL, + 0); /* ack B's records */ + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + + /* C sees nothing. */ + consumer = new_share_consumer(ctx.bootstraps, "sg-ack-atomicity"); + subscribe_topics(consumer, &topic, 1); + consumed_c = test_share_consume_msgs(consumer, 1, 10, 500, NULL, 0); + TEST_SAY("ack_atomicity: C consumed %d/0 (should be 0)\n", consumed_c); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed_a == msgcnt && consumed_b == msgcnt && + consumed_c == 0, + "Expected A=%d B=%d C=0, got A=%d B=%d C=%d", msgcnt, + msgcnt, consumed_a, consumed_b, consumed_c); + SUB_TEST_PASS(); +} + +/** + * @brief Leader change mid-session -> client redirects. + * + * Consumer fetches from broker 1, then we move the partition leader + * to broker 2. The next ShareFetch to broker 1 gets + * NOT_LEADER_OR_FOLLOWER; the client refreshes metadata and + * continues from broker 2. + */ +static void do_test_not_leader_or_follower_redirect(void) { + const char *topic = "kip932_not_leader"; + const int msgcnt = 6; + test_ctx_t ctx; + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST(); + ctx = test_ctx_new(); + + /* Two fetch rounds: 3 records each. */ + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 3); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, 1); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + consumer = new_share_consumer(ctx.bootstraps, "sg-not-leader"); + subscribe_topics(consumer, &topic, 1); + + consumed = test_share_consume_msgs(consumer, 3, 40, 500, NULL, 0); + TEST_SAY("not_leader: consumed %d/3 from broker 1\n", consumed); + + /* Move leader to broker 2 between fetches. */ + rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, 2); + rd_kafka_mock_sharegroup_set_max_record_locks(ctx.mcluster, 0); + + /* Client should handle the redirect transparently. */ + consumed += test_share_consume_msgs(consumer, 3, 60, 500, NULL, 0); + TEST_SAY("not_leader: total consumed %d/6\n", consumed); + + rd_kafka_share_consumer_close(consumer); + rd_kafka_share_destroy(consumer); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == msgcnt, "Expected consumed=%d, got %d", msgcnt, + consumed); + SUB_TEST_PASS(); +} + +/* =================================================================== + * Test runner + * =================================================================== */ + +int main_0157_share_consumer_ack_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + test_timeout_set(200); + + /* Positive scenarios */ + do_test_implicit_ack_no_redelivery(); + do_test_implicit_ack_with_new_records(); + do_test_implicit_ack_cross_consumer(); + do_test_implicit_ack_multi_partition(); + do_test_implicit_ack_multiple_rounds(); + do_test_implicit_ack_single_record(); + do_test_implicit_ack_large_batch(); + do_test_implicit_ack_multi_topic(); + do_test_implicit_ack_multi_msgset(); + + /* Negative scenarios */ + do_test_crash_before_ack_redelivery(); + do_test_crash_then_ack_stops_redelivery(); + do_test_session_expiry_invalidates_ack(); + do_test_max_delivery_without_ack(); + do_test_sharefetch_error_drops_ack(); + do_test_forgotten_topic_releases_not_acks(); + do_test_multi_consumer_cascade_crash(); + do_test_lock_expiry_before_ack(); + do_test_empty_topic_no_ack_side_effects(); + do_test_coordinator_failover_ack_recovery(); + + /* Ack validation */ + do_test_ack_after_lock_expiry_redelivers(); + do_test_ack_success_advances_spso(); + + do_test_final_fetch_no_acquisition(); + do_test_ack_after_log_retention_silent(); + do_test_ack_atomicity_lock_expiry(); + do_test_not_leader_or_follower_redirect(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0158-share_consumer_transactions_mock.c b/lib/librdkafka-2.15.0/tests/0158-share_consumer_transactions_mock.c new file mode 100644 index 00000000000..d71197cf533 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0158-share_consumer_transactions_mock.c @@ -0,0 +1,467 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "rdkafka.h" + +#include "../src/rdkafka_proto.h" + +/** + * @name Share Group Transactions mock broker tests. + * + * Exercises transactional produce + share consume via mock broker + * for both read_uncommitted and read_committed isolation levels. + */ + +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_t *txn_producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new(const char *txn_id) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(3, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + /* Set auto.offset.reset=earliest so tests that produce + * before consuming see all records. */ + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + /* Non-transactional producer */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + /* Transactional producer */ + if (txn_id) { + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "transactional.id", txn_id); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + ctx.txn_producer = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, + sizeof(errstr)); + TEST_ASSERT(ctx.txn_producer != NULL, + "Failed to create txn producer: %s", errstr); + TEST_CALL_ERROR__( + rd_kafka_init_transactions(ctx.txn_producer, 10000)); + } + + return ctx; +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->txn_producer) + rd_kafka_destroy(ctx->txn_producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static void produce_txn_messages(rd_kafka_t *txn_producer, + const char *topic, + int msgcnt, + rd_bool_t commit) { + int i; + + TEST_CALL_ERROR__(rd_kafka_begin_transaction(txn_producer)); + + for (i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "txn-%s-%d", topic, i); + TEST_ASSERT(rd_kafka_producev( + txn_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Txn produce failed"); + } + + rd_kafka_flush(txn_producer, 5000); + + if (commit) + TEST_CALL_ERROR__( + rd_kafka_commit_transaction(txn_producer, 10000)); + else + TEST_CALL_ERROR__( + rd_kafka_abort_transaction(txn_producer, 10000)); +} + +static rd_kafka_share_t *create_share_consumer(const char *bootstraps, + const char *group_id) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer: %s", + errstr); + return rkshare; +} + +static void subscribe_topics(rd_kafka_share_t *share_c, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *tpl = + rd_kafka_topic_partition_list_new(topic_cnt); + int i; + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(tpl, topics[i], + RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(share_c, tpl), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(tpl); +} + + + +/** + * @brief Committed txn data is delivered in read_uncommitted mode. + */ +static void do_test_txn_committed_read_uncommitted(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-committed-ru"); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + produce_txn_messages(ctx.txn_producer, topic, 3, rd_true); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-committed-ru"); + subscribe_topics(share_c, &topic, 1); + consumed = test_share_consume_msgs(share_c, 3, 50, 500, NULL, 0); + TEST_ASSERT(consumed == 3, "Expected 3 consumed, got %d", consumed); + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/** + * @brief Aborted txn data is also delivered in read_uncommitted mode. + * + * In read_uncommitted, the broker does NOT filter aborted data. + */ +static void do_test_txn_aborted_read_uncommitted(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-aborted-ru"); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + produce_txn_messages(ctx.txn_producer, topic, 3, rd_false); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-aborted-ru"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 3, 50, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 3, + "Expected 3 consumed (read_uncommitted sees aborted), " + "got %d", + consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Mixed non-txn + committed txn data in read_uncommitted. + */ +static void do_test_txn_mixed_read_uncommitted(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-mixed-ru"); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 2); + produce_txn_messages(ctx.txn_producer, topic, 3, rd_true); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-mixed-ru"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 5, 50, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 5, "Expected 5 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Committed txn data is delivered in read_committed mode. + */ +static void do_test_txn_committed_read_committed(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-committed-rc"); + + rd_kafka_mock_sharegroup_set_isolation_level(ctx.mcluster, 1); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + produce_txn_messages(ctx.txn_producer, topic, 3, rd_true); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-committed-rc"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 3, 50, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 3, "Expected 3 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Aborted txn data is filtered in read_committed mode. + * + * Share share_c should receive 0 records. + */ +static void do_test_txn_aborted_read_committed(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-aborted-rc"); + + rd_kafka_mock_sharegroup_set_isolation_level(ctx.mcluster, 1); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + produce_txn_messages(ctx.txn_producer, topic, 3, rd_false); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-aborted-rc"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 0, 15, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 0, + "Expected 0 consumed (aborted data filtered), got %d", + consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Mixed committed + aborted + non-txn in read_committed mode. + * + * 2 non-txn + 3 committed + 3 aborted + 2 non-txn. + * Expected: 7 (2 non-txn + 3 committed data + 2 trailing non-txn). + * Aborted data is filtered/archived, control records are skipped via + * GAP(0) ack, so the trailing non-txn records are now reachable. + */ +static void do_test_txn_mixed_read_committed(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-mixed-rc"); + + rd_kafka_mock_sharegroup_set_isolation_level(ctx.mcluster, 1); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 2); + produce_txn_messages(ctx.txn_producer, topic, 3, rd_true); + produce_txn_messages(ctx.txn_producer, topic, 3, rd_false); + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 2); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-mixed-rc"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 7, 60, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 7, + "Expected 7 consumed (2 non-txn + 3 committed + " + "2 trailing non-txn), got %d", + consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Non-txn only data in read_committed mode works normally. + * + * No transactions involved — LSO = end_offset, no filtering. + */ +static void do_test_txn_nontxn_read_committed(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new(NULL); + + rd_kafka_mock_sharegroup_set_isolation_level(ctx.mcluster, 1); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, 5); + + share_c = create_share_consumer(ctx.bootstraps, "sg-txn-nontxn-rc"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 5, 50, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 5, "Expected 5 consumed, got %d", consumed); + SUB_TEST_PASS(); +} + +/** + * @brief Abort then commit in read_committed. + * + * Log layout: + * offset 0-2: aborted data -> ARCHIVED by broker + * offset 3: ABORT control -> skipped via GAP(0) ack + * offset 4-6: committed data -> delivered + * offset 7: COMMIT control -> skipped via GAP(0) ack + * + * Consumer skips control records via GAP(0) ack and receives the + * 3 committed data records. + */ +static void do_test_txn_abort_then_commit_read_committed(void) { + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + test_ctx_t ctx; + rd_kafka_share_t *share_c; + int consumed; + + SUB_TEST_QUICK(); + ctx = test_ctx_new("txn-abort-commit-rc"); + + rd_kafka_mock_sharegroup_set_isolation_level(ctx.mcluster, 1); + + TEST_CALL_ERR__(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1)); + + produce_txn_messages(ctx.txn_producer, topic, 3, rd_false); + produce_txn_messages(ctx.txn_producer, topic, 3, rd_true); + + share_c = + create_share_consumer(ctx.bootstraps, "sg-txn-abort-commit-rc"); + subscribe_topics(share_c, &topic, 1); + + consumed = test_share_consume_msgs(share_c, 3, 50, 500, NULL, 0); + + test_share_consumer_close(share_c); + test_share_destroy(share_c); + test_ctx_destroy(&ctx); + + TEST_ASSERT(consumed == 3, + "Expected 3 consumed (committed data after abort), " + "got %d", + consumed); + SUB_TEST_PASS(); +} + + +int main_0158_share_consumer_transactions_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + test_timeout_set(1500); + + do_test_txn_committed_read_uncommitted(); + + do_test_txn_aborted_read_uncommitted(); + + do_test_txn_mixed_read_uncommitted(); + + do_test_txn_committed_read_committed(); + + do_test_txn_aborted_read_committed(); + + do_test_txn_mixed_read_committed(); + + do_test_txn_nontxn_read_committed(); + + do_test_txn_abort_then_commit_read_committed(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0170-share_consumer_subscription.c b/lib/librdkafka-2.15.0/tests/0170-share_consumer_subscription.c new file mode 100644 index 00000000000..26fd4a3bb0a --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0170-share_consumer_subscription.c @@ -0,0 +1,2192 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "testshared.h" +#include "rdkafka.h" + +/** + * @brief Share consumer subscription tests using operation-based framework. + * + * This test file uses a declarative, operation-based framework where tests + * are defined as sequences of operations. The framework handles: + * - Automatic topic name generation + * - Topic creation and deletion + * - Consumer creation and destruction + * - Message production and consumption + * - Subscription verification + */ + + +#define MAX_TOPICS 20 +#define MAX_CONSUMERS 4 +#define MAX_OPS 50 + +/** Common producer reused across all tests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all tests. */ +static rd_kafka_t *common_admin; + +/** + * @brief Operation types for subscription tests + */ +typedef enum { + TEST_OP_END = 0, /**< End of operations marker */ + TEST_OP_SUBSCRIBE, /**< Subscribe to N new topics */ + TEST_OP_SUBSCRIBE_ADD, /**< Add N topics to existing subscription */ + TEST_OP_UNSUBSCRIBE, /**< Unsubscribe from all topics */ + TEST_OP_RESUBSCRIBE, /**< Replace subscription with N new topics */ + TEST_OP_PRODUCE, /**< Produce to specified topic set */ + TEST_OP_CONSUME, /**< Consume messages */ + TEST_OP_VERIFY_SUB_CNT, /**< Verify subscription count */ + TEST_OP_DELETE_TOPIC, /**< Delete topic by index */ + TEST_OP_WAIT, /**< Wait for specified milliseconds */ + TEST_OP_CREATE_CONSUMER, /**< Create additional consumer */ + TEST_OP_POLL_NO_SUB, /**< Poll without subscription (edge case) */ + TEST_OP_CREATE_TOPIC, /**< Create subscribed topics that weren't created + */ + TEST_OP_SUBSCRIBE_EXISTING, /**< Subscribe to already created topics */ + TEST_OP_PRODUCE_TO_TOPIC, /**< Produce to specific topic index */ + TEST_OP_SUBSCRIBE_EMPTY, /**< Subscribe with empty list (== + unsubscribe) */ +} test_op_type_t; + +/** + * @brief Flags for operations + */ +typedef enum { + TEST_OP_F_NONE = 0, + TEST_OP_F_SKIP_TOPIC_CREATE = 1 << 0, /**< Don't create topics */ + TEST_OP_F_PRODUCE_TO_OLD = 1 << 1, /**< Produce to old subscription */ + TEST_OP_F_VERIFY_NO_OLD_MSGS = 1 << 2, /**< Verify no old messages */ +} test_op_flags_t; + +/** + * @brief Single operation in a test scenario + */ +typedef struct { + test_op_type_t op; /**< Operation type */ + int topic_cnt; /**< Number of topics (SUBSCRIBE/RESUBSCRIBE) */ + int msgs_per_topic; /**< Messages per topic (PRODUCE) */ + int expected_msgs; /**< Expected message count (CONSUME, -1=any) */ + int expected_sub_cnt; /**< Expected subscription count (VERIFY_SUB_CNT) + */ + int topic_idx; /**< Topic index (DELETE_TOPIC) */ + int wait_ms; /**< Wait time (WAIT) */ + int consumer_idx; /**< Consumer index (multi-consumer) */ + int repeat_cnt; /**< Repeat count (SUBSCRIBE/UNSUBSCRIBE) */ + test_op_flags_t flags; /**< Operation flags */ +} test_op_t; + +/** + * @brief Test scenario configuration + */ +typedef struct { + const char *name; /**< Test name for logging */ + int consumer_cnt; /**< Number of consumers (default: 1) */ + test_op_t ops[MAX_OPS]; /**< Operations, terminated by TEST_OP_END */ +} test_scenario_t; + +/** + * @brief Runtime state for test execution + */ +typedef struct { + /* Consumers */ + rd_kafka_share_t *consumers[MAX_CONSUMERS]; + int consumer_cnt; + + /* Topics: all created topics */ + char *all_topics[MAX_TOPICS]; + int all_topic_cnt; + int msgs_produced[MAX_TOPICS]; /**< Messages produced per topic */ + + /* Current subscription tracking per consumer */ + int sub_start_idx[MAX_CONSUMERS]; /**< Start index in all_topics */ + int sub_count[MAX_CONSUMERS]; /**< Count of subscribed topics */ + + /* Previous subscription (for RESUBSCRIBE verification) */ + int old_sub_start_idx; + int old_sub_count; + + /* Group name */ + char group_name[128]; +} sub_test_state_t; + + +#define SUBSCRIBE(n) \ + { .op = TEST_OP_SUBSCRIBE, .topic_cnt = (n), .repeat_cnt = 1 } +#define SUBSCRIBE_REPEAT(n, r) \ + { .op = TEST_OP_SUBSCRIBE, .topic_cnt = (n), .repeat_cnt = (r) } +#define SUBSCRIBE_ADD(n) \ + { .op = TEST_OP_SUBSCRIBE_ADD, .topic_cnt = (n) } +#define SUBSCRIBE_NO_CREATE(n) \ + { \ + .op = TEST_OP_SUBSCRIBE, .topic_cnt = (n), .repeat_cnt = 1, \ + .flags = TEST_OP_F_SKIP_TOPIC_CREATE \ + } +#define UNSUBSCRIBE() \ + { .op = TEST_OP_UNSUBSCRIBE, .repeat_cnt = 1 } +#define UNSUBSCRIBE_REPEAT(r) \ + { .op = TEST_OP_UNSUBSCRIBE, .repeat_cnt = (r) } +#define RESUBSCRIBE(n) \ + { .op = TEST_OP_RESUBSCRIBE, .topic_cnt = (n) } +#define PRODUCE(msgs) \ + { .op = TEST_OP_PRODUCE, .msgs_per_topic = (msgs) } +#define PRODUCE_TO_OLD(msgs) \ + { \ + .op = TEST_OP_PRODUCE, .msgs_per_topic = (msgs), \ + .flags = TEST_OP_F_PRODUCE_TO_OLD \ + } +#define PRODUCE_TO_TOPIC(idx, msgs) \ + { \ + .op = TEST_OP_PRODUCE_TO_TOPIC, .topic_idx = (idx), \ + .msgs_per_topic = (msgs) \ + } +#define CONSUME(expected) \ + { .op = TEST_OP_CONSUME, .expected_msgs = (expected) } +#define CONSUME_VERIFY_NO_OLD(expected) \ + { \ + .op = TEST_OP_CONSUME, .expected_msgs = (expected), \ + .flags = TEST_OP_F_VERIFY_NO_OLD_MSGS \ + } +#define CONSUME_ANY() \ + { .op = TEST_OP_CONSUME, .expected_msgs = -1 } +#define VERIFY_SUB(cnt) \ + { .op = TEST_OP_VERIFY_SUB_CNT, .expected_sub_cnt = (cnt) } +#define DELETE_TOPIC(idx) \ + { .op = TEST_OP_DELETE_TOPIC, .topic_idx = (idx) } +#define WAIT_MS(ms) \ + { .op = TEST_OP_WAIT, .wait_ms = (ms) } +#define CREATE_CONSUMER(idx) \ + { .op = TEST_OP_CREATE_CONSUMER, .consumer_idx = (idx) } +#define CREATE_TOPIC(n) \ + { .op = TEST_OP_CREATE_TOPIC, .topic_cnt = (n) } +#define SUBSCRIBE_EXISTING() \ + { .op = TEST_OP_SUBSCRIBE_EXISTING, .repeat_cnt = 1 } +#define POLL_NO_SUB() \ + { .op = TEST_OP_POLL_NO_SUB } +#define SUBSCRIBE_EMPTY() \ + { .op = TEST_OP_SUBSCRIBE_EMPTY } +#define TEST_OPS_END() \ + { .op = TEST_OP_END } + + +/** + * @brief Create a new topic with auto-generated name + */ +static const char *state_create_topic(sub_test_state_t *state, + rd_bool_t wait_exists) { + char name[128]; + + TEST_ASSERT(state->all_topic_cnt < MAX_TOPICS, + "Too many topics created"); + + rd_snprintf(name, sizeof(name), "0170-t%d", state->all_topic_cnt); + state->all_topics[state->all_topic_cnt] = + rd_strdup(test_mk_topic_name(name, 1)); + + if (wait_exists) { + test_create_topic_wait_exists( + NULL, state->all_topics[state->all_topic_cnt], 1, -1, + 30000); + } + + state->msgs_produced[state->all_topic_cnt] = 0; + return state->all_topics[state->all_topic_cnt++]; +} + +/** + * @brief Execute TEST_OP_SUBSCRIBE + */ +static void exec_subscribe(sub_test_state_t *state, const test_op_t *op) { + rd_kafka_topic_partition_list_t *tlist; + int cidx = op->consumer_idx; + int i, r; + + TEST_SAY(" SUBSCRIBE: %d topic(s), repeat=%d, consumer=%d\n", + op->topic_cnt, op->repeat_cnt, cidx); + + /* Save old subscription for RESUBSCRIBE verification */ + state->old_sub_start_idx = state->sub_start_idx[cidx]; + state->old_sub_count = state->sub_count[cidx]; + + /* Track new subscription */ + state->sub_start_idx[cidx] = state->all_topic_cnt; + state->sub_count[cidx] = op->topic_cnt; + + /* Create topics and build subscription list */ + tlist = rd_kafka_topic_partition_list_new(op->topic_cnt); + for (i = 0; i < op->topic_cnt; i++) { + const char *topic = state_create_topic( + state, !(op->flags & TEST_OP_F_SKIP_TOPIC_CREATE)); + rd_kafka_topic_partition_list_add(tlist, topic, + RD_KAFKA_PARTITION_UA); + } + + /* Subscribe (possibly multiple times) */ + for (r = 0; r < op->repeat_cnt; r++) { + TEST_CALL_ERR__( + rd_kafka_share_subscribe(state->consumers[cidx], tlist)); + } + + rd_kafka_topic_partition_list_destroy(tlist); +} + +/** + * @brief Execute TEST_OP_SUBSCRIBE_ADD (incremental - add to existing + * subscription) + */ +static void exec_subscribe_add(sub_test_state_t *state, const test_op_t *op) { + rd_kafka_topic_partition_list_t *tlist; + int cidx = op->consumer_idx; + int i; + int new_start = state->all_topic_cnt; + + TEST_SAY( + " SUBSCRIBE_ADD: adding %d topic(s) to existing %d, consumer=%d\n", + op->topic_cnt, state->sub_count[cidx], cidx); + + /* Build subscription list including existing + new topics */ + tlist = rd_kafka_topic_partition_list_new(state->sub_count[cidx] + + op->topic_cnt); + + /* Add existing subscribed topics */ + for (i = 0; i < state->sub_count[cidx]; i++) { + int idx = state->sub_start_idx[cidx] + i; + rd_kafka_topic_partition_list_add(tlist, state->all_topics[idx], + RD_KAFKA_PARTITION_UA); + } + + /* Create and add new topics */ + for (i = 0; i < op->topic_cnt; i++) { + const char *topic = state_create_topic(state, rd_true); + rd_kafka_topic_partition_list_add(tlist, topic, + RD_KAFKA_PARTITION_UA); + } + + TEST_CALL_ERR__( + rd_kafka_share_subscribe(state->consumers[cidx], tlist)); + + /* Update subscription tracking - topics are now spread across ranges */ + state->sub_start_idx[cidx] = new_start - state->sub_count[cidx]; + state->sub_count[cidx] += op->topic_cnt; + + rd_kafka_topic_partition_list_destroy(tlist); +} + +/** + * @brief Execute TEST_OP_CREATE_TOPIC (create topics that weren't created) + */ +static void exec_create_topic(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + int i; + + TEST_SAY(" CREATE_TOPIC: creating subscribed topics for consumer=%d\n", + cidx); + + /* Create the topics that were subscribed to but not yet created */ + for (i = 0; i < state->sub_count[cidx]; i++) { + int idx = state->sub_start_idx[cidx] + i; + if (state->all_topics[idx]) { + test_create_topic_wait_exists( + NULL, state->all_topics[idx], 1, -1, 30000); + } + } +} + +/** + * @brief Execute TEST_OP_SUBSCRIBE_EXISTING (subscribe to all created topics) + */ +static void exec_subscribe_existing(sub_test_state_t *state, + const test_op_t *op) { + rd_kafka_topic_partition_list_t *tlist; + int cidx = op->consumer_idx; + int i; + + TEST_SAY(" SUBSCRIBE_EXISTING: %d topic(s), consumer=%d\n", + state->all_topic_cnt, cidx); + + tlist = rd_kafka_topic_partition_list_new(state->all_topic_cnt); + + for (i = 0; i < state->all_topic_cnt; i++) { + rd_kafka_topic_partition_list_add(tlist, state->all_topics[i], + RD_KAFKA_PARTITION_UA); + } + + TEST_CALL_ERR__( + rd_kafka_share_subscribe(state->consumers[cidx], tlist)); + + state->sub_start_idx[cidx] = 0; + state->sub_count[cidx] = state->all_topic_cnt; + + rd_kafka_topic_partition_list_destroy(tlist); +} + +/** + * @brief Execute TEST_OP_PRODUCE_TO_TOPIC (produce to specific topic by index) + */ +static void exec_produce_to_topic(sub_test_state_t *state, + const test_op_t *op) { + int cidx = op->consumer_idx; + int idx = state->sub_start_idx[cidx] + op->topic_idx; + + TEST_ASSERT(op->topic_idx < state->sub_count[cidx], + "Topic index %d out of range (sub_count=%d)", op->topic_idx, + state->sub_count[cidx]); + + TEST_SAY(" PRODUCE_TO_TOPIC: %d msgs to topic[%d] (%s)\n", + op->msgs_per_topic, op->topic_idx, state->all_topics[idx]); + + test_produce_msgs_simple(common_producer, state->all_topics[idx], 0, + op->msgs_per_topic); + state->msgs_produced[idx] += op->msgs_per_topic; +} + +/** + * @brief Execute TEST_OP_RESUBSCRIBE (replace subscription with new topics) + */ +static void exec_resubscribe(sub_test_state_t *state, const test_op_t *op) { + rd_kafka_topic_partition_list_t *tlist; + int cidx = op->consumer_idx; + int i; + + TEST_SAY(" RESUBSCRIBE: %d new topic(s), consumer=%d\n", op->topic_cnt, + cidx); + + /* Save old subscription */ + state->old_sub_start_idx = state->sub_start_idx[cidx]; + state->old_sub_count = state->sub_count[cidx]; + + /* Track new subscription */ + state->sub_start_idx[cidx] = state->all_topic_cnt; + state->sub_count[cidx] = op->topic_cnt; + + /* Create new topics */ + tlist = rd_kafka_topic_partition_list_new(op->topic_cnt); + for (i = 0; i < op->topic_cnt; i++) { + const char *topic = state_create_topic(state, rd_true); + rd_kafka_topic_partition_list_add(tlist, topic, + RD_KAFKA_PARTITION_UA); + } + + TEST_CALL_ERR__( + rd_kafka_share_subscribe(state->consumers[cidx], tlist)); + rd_kafka_topic_partition_list_destroy(tlist); +} + +/** + * @brief Execute TEST_OP_UNSUBSCRIBE + */ +static void exec_unsubscribe(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + int r; + + TEST_SAY(" UNSUBSCRIBE: repeat=%d, consumer=%d\n", op->repeat_cnt, + cidx); + + for (r = 0; r < op->repeat_cnt; r++) { + TEST_CALL_ERR__( + rd_kafka_share_unsubscribe(state->consumers[cidx])); + } + + state->sub_count[cidx] = 0; +} + +/** + * @brief Execute TEST_OP_SUBSCRIBE_EMPTY + * + * Subscribe with an empty topic list is equivalent to unsubscribe: + * must return NO_ERROR and clear the subscription flag. + */ +static void exec_subscribe_empty(sub_test_state_t *state, const test_op_t *op) { + rd_kafka_topic_partition_list_t *tlist; + int cidx = op->consumer_idx; + + TEST_SAY(" SUBSCRIBE_EMPTY: consumer=%d\n", cidx); + + tlist = rd_kafka_topic_partition_list_new(0); + TEST_CALL_ERR__( + rd_kafka_share_subscribe(state->consumers[cidx], tlist)); + rd_kafka_topic_partition_list_destroy(tlist); + + state->sub_count[cidx] = 0; +} + +/** + * @brief Execute TEST_OP_PRODUCE + */ +static void exec_produce(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + int start_idx, count, i; + + if (op->flags & TEST_OP_F_PRODUCE_TO_OLD) { + start_idx = state->old_sub_start_idx; + count = state->old_sub_count; + TEST_SAY(" PRODUCE: %d msgs/topic to OLD %d topic(s)\n", + op->msgs_per_topic, count); + } else { + start_idx = state->sub_start_idx[cidx]; + count = state->sub_count[cidx]; + TEST_SAY(" PRODUCE: %d msgs/topic to %d topic(s)\n", + op->msgs_per_topic, count); + } + + for (i = 0; i < count; i++) { + int idx = start_idx + i; + test_produce_msgs_simple(common_producer, + state->all_topics[idx], 0, + op->msgs_per_topic); + state->msgs_produced[idx] += op->msgs_per_topic; + } +} + +/** + * @brief Execute TEST_OP_CONSUME + */ +static void exec_consume(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + int start_idx = state->sub_start_idx[cidx]; + int count = state->sub_count[cidx]; + const char *topics[MAX_TOPICS]; + int i, consumed; + + /* Build expected topics array */ + for (i = 0; i < count; i++) { + topics[i] = state->all_topics[start_idx + i]; + } + + if (op->expected_msgs >= 0) { + TEST_SAY(" CONSUME: expecting %d msgs from %d topic(s)\n", + op->expected_msgs, count); + consumed = test_share_consume_msgs( + state->consumers[cidx], op->expected_msgs, 25, 3000, + count > 0 ? topics : NULL, count); + + if (op->flags & TEST_OP_F_VERIFY_NO_OLD_MSGS) { + TEST_ASSERT(consumed >= 0, + "Received message from old subscription!"); + } + TEST_ASSERT(consumed == op->expected_msgs, + "Expected %d messages, got %d", op->expected_msgs, + consumed); + } else { + /* Consume any available */ + TEST_SAY(" CONSUME: any available from %d topic(s)\n", count); + test_share_consume_msgs(state->consumers[cidx], 100, 10, 2000, + count > 0 ? topics : NULL, count); + } +} + +/** + * @brief Execute TEST_OP_VERIFY_SUB_CNT + */ +static void exec_verify_sub_cnt(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + rd_kafka_topic_partition_list_t *sub; + + TEST_SAY(" VERIFY_SUB_CNT: expecting %d, consumer=%d\n", + op->expected_sub_cnt, cidx); + + sub = test_get_subscription(state->consumers[cidx]); + TEST_ASSERT(sub->cnt == op->expected_sub_cnt, + "Expected %d subscriptions, got %d", op->expected_sub_cnt, + sub->cnt); + rd_kafka_topic_partition_list_destroy(sub); +} + +/** + * @brief Execute TEST_OP_DELETE_TOPIC + */ +static void exec_delete_topic(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + int idx = state->sub_start_idx[cidx] + op->topic_idx; + + TEST_ASSERT(op->topic_idx < state->sub_count[cidx], + "Topic index %d out of range (sub_count=%d)", op->topic_idx, + state->sub_count[cidx]); + + TEST_SAY(" DELETE_TOPIC: index %d (%s)\n", op->topic_idx, + state->all_topics[idx]); + + /* Actually delete the topic */ + test_DeleteTopics_simple(common_admin, NULL, &state->all_topics[idx], 1, + NULL); +} + +/** + * @brief Execute TEST_OP_WAIT + */ +static void exec_wait(sub_test_state_t *state, const test_op_t *op) { + TEST_SAY(" WAIT: %d ms\n", op->wait_ms); + rd_sleep(op->wait_ms / 1000); + if (op->wait_ms % 1000) + rd_usleep((op->wait_ms % 1000) * 1000, NULL); +} + +/** + * @brief Execute TEST_OP_CREATE_CONSUMER + */ +static void exec_create_consumer(sub_test_state_t *state, const test_op_t *op) { + int cidx = op->consumer_idx; + + TEST_SAY(" CREATE_CONSUMER: index %d\n", cidx); + + TEST_ASSERT(cidx < MAX_CONSUMERS, "Consumer index out of range"); + TEST_ASSERT(state->consumers[cidx] == NULL, + "Consumer %d already exists", cidx); + + state->consumers[cidx] = + test_create_share_consumer(state->group_name, NULL); + if (cidx >= state->consumer_cnt) + state->consumer_cnt = cidx + 1; +} + +/** + * @brief Execute TEST_OP_POLL_NO_SUB + * + * consume_batch must surface RD_KAFKA_RESP_ERR__STATE with a "not + * subscribed" message when no subscription is active. + */ +static void exec_poll_no_sub(sub_test_state_t *state, const test_op_t *op) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + size_t rcvd = 0; + int cidx = op->consumer_idx; + + TEST_SAY(" POLL_NO_SUB: consumer=%d\n", cidx); + + err = rd_kafka_share_poll(state->consumers[cidx], 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(err != NULL, + "POLL_NO_SUB consumer=%d: expected error, got NULL", cidx); + TEST_ASSERT(rd_kafka_error_code(err) == RD_KAFKA_RESP_ERR__STATE, + "POLL_NO_SUB consumer=%d: expected __STATE, got: %s", cidx, + rd_kafka_err2name(rd_kafka_error_code(err))); + TEST_ASSERT(strstr(rd_kafka_error_string(err), "not subscribed"), + "POLL_NO_SUB consumer=%d: expected 'not subscribed' " + "substring, got: %s", + cidx, rd_kafka_error_string(err)); + TEST_ASSERT(rcvd == 0, + "POLL_NO_SUB consumer=%d: expected 0 messages, got %zu", + cidx, rcvd); + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); +} + +/** + * @brief Initialize test state + */ +static void state_init(sub_test_state_t *state, + const test_scenario_t *scenario) { + int i; + + memset(state, 0, sizeof(*state)); + + /* Append a per-invocation unique suffix so each run gets a + * fresh share-group on the broker (avoids collisions across + * re-runs or parallel runs against the same cluster). */ + rd_snprintf(state->group_name, sizeof(state->group_name), + "share-%s-rnd%" PRIx64, scenario->name, test_id_generate()); + TEST_SAY("Scenario '%s' using group '%s'\n", scenario->name, + state->group_name); + + state->consumer_cnt = + scenario->consumer_cnt > 0 ? scenario->consumer_cnt : 1; + + /* Create initial consumers */ + for (i = 0; i < state->consumer_cnt; i++) { + state->consumers[i] = + test_create_share_consumer(state->group_name, NULL); + } + + /* Set group offset to earliest */ + test_share_set_auto_offset_reset(state->group_name, "earliest"); +} + +/** + * @brief Cleanup test state + */ +static void state_cleanup(sub_test_state_t *state) { + int i; + + for (i = 0; i < state->all_topic_cnt; i++) { + if (state->all_topics[i]) { + rd_free(state->all_topics[i]); + } + } + + /* Destroy all consumers */ + for (i = 0; i < MAX_CONSUMERS; i++) { + if (state->consumers[i]) { + test_share_consumer_close(state->consumers[i]); + test_share_destroy(state->consumers[i]); + } + } +} + +/** + * @brief Run a test scenario + */ +static void do_test_scenario(const test_scenario_t *scenario) { + sub_test_state_t state; + int op_idx; + + SUB_TEST(); + + state_init(&state, scenario); + + /* Execute operations */ + for (op_idx = 0; scenario->ops[op_idx].op != TEST_OP_END; op_idx++) { + const test_op_t *op = &scenario->ops[op_idx]; + + switch (op->op) { + case TEST_OP_SUBSCRIBE: + exec_subscribe(&state, op); + break; + case TEST_OP_SUBSCRIBE_ADD: + exec_subscribe_add(&state, op); + break; + case TEST_OP_RESUBSCRIBE: + exec_resubscribe(&state, op); + break; + case TEST_OP_UNSUBSCRIBE: + exec_unsubscribe(&state, op); + break; + case TEST_OP_PRODUCE: + exec_produce(&state, op); + break; + case TEST_OP_PRODUCE_TO_TOPIC: + exec_produce_to_topic(&state, op); + break; + case TEST_OP_CONSUME: + exec_consume(&state, op); + break; + case TEST_OP_VERIFY_SUB_CNT: + exec_verify_sub_cnt(&state, op); + break; + case TEST_OP_DELETE_TOPIC: + exec_delete_topic(&state, op); + break; + case TEST_OP_WAIT: + exec_wait(&state, op); + break; + case TEST_OP_CREATE_CONSUMER: + exec_create_consumer(&state, op); + break; + case TEST_OP_CREATE_TOPIC: + exec_create_topic(&state, op); + break; + case TEST_OP_SUBSCRIBE_EXISTING: + exec_subscribe_existing(&state, op); + break; + case TEST_OP_POLL_NO_SUB: + exec_poll_no_sub(&state, op); + break; + case TEST_OP_SUBSCRIBE_EMPTY: + exec_subscribe_empty(&state, op); + break; + default: + TEST_FAIL("Unknown operation: %d", op->op); + } + } + + state_cleanup(&state); + + SUB_TEST_PASS(); +} + + +/** + * Basic subscription tests + */ +static const test_scenario_t test_single_subscribe = { + .name = "single-subscribe", + .ops = {SUBSCRIBE(2), PRODUCE(5), VERIFY_SUB(2), CONSUME(10), + TEST_OPS_END()}}; + +static const test_scenario_t test_single_unsubscribe = { + .name = "single-unsubscribe", + .ops = {SUBSCRIBE(2), PRODUCE(5), CONSUME(10), UNSUBSCRIBE(), VERIFY_SUB(0), + TEST_OPS_END()}}; + +static const test_scenario_t test_repeated_subscribe = { + .name = "repeated-subscribe-no-duplicates", + .ops = {SUBSCRIBE_REPEAT(2, 3), /* Subscribe 3 times to same topics */ + PRODUCE(5), VERIFY_SUB(2), /* Should still be 2, not 6 */ + CONSUME(10), TEST_OPS_END()}}; + +static const test_scenario_t test_repeated_unsubscribe = { + .name = "repeated-unsubscribe-no-error", + .ops = {SUBSCRIBE(2), PRODUCE(5), CONSUME(10), + UNSUBSCRIBE_REPEAT(3), /* Unsubscribe 3 times */ + VERIFY_SUB(0), TEST_OPS_END()}}; + +/** + * Subscription replacement tests + */ +/* TODO KIP-932: test_topic_switch might be incorrect. Verify and remove. */ +// static const test_scenario_t test_topic_switch = { +// .name = "topic-switch", +// .ops = {SUBSCRIBE(2), PRODUCE(10), CONSUME_ANY(), +// RESUBSCRIBE(2), /* Switch to 2 new topics */ +// PRODUCE(10), /* Produce to new topics */ +// PRODUCE_TO_OLD(5), /* Produce to old topics */ +// CONSUME_VERIFY_NO_OLD(20), /* Should only get new topic msgs */ +// TEST_OPS_END()}}; + +static const test_scenario_t test_incremental_subscription = { + .name = "incremental-subscription", + .ops = {/* Start with 1 topic */ + SUBSCRIBE(1), PRODUCE(10), VERIFY_SUB(1), CONSUME(10), + /* Add 1 more topic (now 2 total) */ + SUBSCRIBE_ADD(1), PRODUCE(10), VERIFY_SUB(2), + CONSUME(20), /* 10 from each of 2 topics */ + /* Add 1 more topic (now 3 total) */ + SUBSCRIBE_ADD(1), PRODUCE(10), VERIFY_SUB(3), + CONSUME(30), /* 10 from each of 3 topics */ + TEST_OPS_END()}}; + +/** + * Edge case tests + */ +static const test_scenario_t test_subscribe_before_topic_exists = { + .name = "subscribe-before-topic-exists", + .ops = {SUBSCRIBE_NO_CREATE(1), /* Subscribe without creating topic */ + CREATE_TOPIC(0), /* Now create the subscribed topic */ + PRODUCE(5), /* Produce to the topic */ + CONSUME(5), /* Should receive all messages */ + TEST_OPS_END()}}; + +static const test_scenario_t test_poll_empty_topic = { + .name = "poll-empty-topic", + .ops = {SUBSCRIBE(1), + /* Don't produce - topic is empty */ + CONSUME(0), /* Should return 0, not error */ + TEST_OPS_END()}}; + +static const test_scenario_t test_poll_no_subscription = { + .name = "poll-no-subscription", + .ops = {POLL_NO_SUB(), /* Poll without subscribing */ + TEST_OPS_END()}}; + +static const test_scenario_t test_poll_after_unsubscribe = { + .name = "poll-after-unsubscribe", + .ops = {SUBSCRIBE(1), PRODUCE(5), CONSUME_ANY(), /* Consume some */ + UNSUBSCRIBE(), POLL_NO_SUB(), /* Poll after unsubscribe */ + TEST_OPS_END()}}; + +/** + * Topic deletion tests + */ +static const test_scenario_t test_topic_deletion = { + .name = "topic-deletion-while-subscribed", + .ops = {SUBSCRIBE(2), PRODUCE(10), CONSUME_ANY(), + DELETE_TOPIC(1), /* Delete second topic */ + WAIT_MS(3000), + PRODUCE_TO_TOPIC(0, 5), /* Produce to remaining topic */ + CONSUME_ANY(), /* Continue consuming from remaining */ + TEST_OPS_END()}}; + +/** + * Stress tests + */ +static const test_scenario_t test_rapid_updates = { + .name = "rapid-subscription-updates", + .ops = {SUBSCRIBE(2), RESUBSCRIBE(1), RESUBSCRIBE(3), UNSUBSCRIBE(), + SUBSCRIBE(2), RESUBSCRIBE(2), UNSUBSCRIBE(), SUBSCRIBE(3), + VERIFY_SUB(3), TEST_OPS_END()}}; + +/** + * @brief Multi-consumer overlap test (standalone) + * + * Two consumers in same group with overlapping subscriptions: + * - Consumer 0: [shared, c0_only] + * - Consumer 1: [shared, c1_only] + * + * This test verifies share group consumers can have overlapping + * subscriptions and both receive messages from shared topics. + */ +static void do_test_multi_consumer_overlap(void) { + const char *group = test_mk_topic_name("share-overlap", 1); + char *shared = rd_strdup(test_mk_topic_name("0170-shared", 1)); + char *c0_only = rd_strdup(test_mk_topic_name("0170-c0only", 1)); + char *c1_only = rd_strdup(test_mk_topic_name("0170-c1only", 1)); + const char *c0_topics[] = {shared, c0_only}; + const char *c1_topics[] = {shared, c1_only}; + rd_kafka_share_t *rkshare0, *rkshare1; + int c0_cnt = 0, c1_cnt = 0; + int attempts; + + SUB_TEST("multi-consumer-overlapping-subscriptions"); + + /* Create topics */ + test_create_topic_wait_exists(NULL, shared, 1, -1, 30000); + test_create_topic_wait_exists(NULL, c0_only, 1, -1, 30000); + test_create_topic_wait_exists(NULL, c1_only, 1, -1, 30000); + + /* Produce messages */ + test_produce_msgs_simple(common_producer, shared, 0, 20); + test_produce_msgs_simple(common_producer, c0_only, 0, 10); + test_produce_msgs_simple(common_producer, c1_only, 0, 10); + + /* Create consumers */ + rkshare0 = test_create_share_consumer(group, NULL); + rkshare1 = test_create_share_consumer(group, NULL); + + /* Set group offset */ + test_share_set_auto_offset_reset(group, "earliest"); + + /* Subscribe with overlapping topics */ + test_share_consumer_subscribe_multi(rkshare0, 2, shared, c0_only); + test_share_consumer_subscribe_multi(rkshare1, 2, shared, c1_only); + + /* Consume - alternate between consumers */ + attempts = 20; + while ((c0_cnt + c1_cnt) < 10 && attempts-- > 0) { + int batch_cnt = 0; + int ret; + + ret = test_share_poll(rkshare0, 2000, c0_topics, 2, &batch_cnt); + TEST_ASSERT(ret >= 0, "C0 wrong topic"); + c0_cnt += batch_cnt; + + batch_cnt = 0; + ret = test_share_poll(rkshare1, 2000, c1_topics, 2, &batch_cnt); + TEST_ASSERT(ret >= 0, "C1 wrong topic"); + c1_cnt += batch_cnt; + } + + TEST_SAY("C0: %d, C1: %d (total: %d)\n", c0_cnt, c1_cnt, + c0_cnt + c1_cnt); + TEST_ASSERT(c0_cnt > 0 || c1_cnt > 0, "no messages received"); + + /* Cleanup */ + test_share_consumer_close(rkshare0); + test_share_consumer_close(rkshare1); + test_share_destroy(rkshare0); + test_share_destroy(rkshare1); + + rd_free(shared); + rd_free(c0_only); + rd_free(c1_only); + + SUB_TEST_PASS(); +} + + +/** + * @brief Topic-level metadata-error policy: when a subscribed topic is + * deleted mid-stream, the resulting metadata error + * (UNKNOWN_TOPIC_OR_PART / UNKNOWN_TOPIC) is treated as a + * transient code by the share consumer — logged internally and + * not surfaced to the application via consume_batch. + * + * Subscribe to a valid topic, prime the share assignment so the rktp + * is on rkt_desp, delete the topic, allow metadata propagation, and + * verify that no topic-level err shows up in subsequent consume_batch + * calls. + */ +static void test_topic_deletion_does_not_surface_error(void) { + const char *topic; + const char *group = "share-topic-delete-no-surface"; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + char errstr[512]; + char *topic_dup; + char *topics_for_delete[1]; + rd_kafka_resp_err_t del_err; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int post_attempts; + rd_bool_t surfaced_topic_err = rd_false; + + SUB_TEST(); + + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) + SUB_TEST_SKIP( + "topic-deletion-does-not-surface-error " + "(broker on Windows)\n"); + + topic = test_mk_topic_name("0170-delete-no-surface", 1); + topic_dup = rd_strdup(topic); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_share_set_auto_offset_reset(group, "earliest"); + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Custom consumer that disables the set_notexists defer window + * (default 30s) so the log-and-drop path runs on the first + * post-delete metadata refresh instead of being deferred. + * Also reduce the metadata refresh interval so a refresh actually + * happens during the post-delete settle window — the default is + * 5 minutes, which would never fire during the test. */ + test_conf_init(&conf, NULL, 60); + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "share.acknowledgement.mode", "explicit", + errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "topic.metadata.propagation.max.ms", "0", + errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "topic.metadata.refresh.interval.ms", "500", + errstr, sizeof(errstr)); + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create share consumer: %s", errstr); + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Prime the share assignment: consume + ACCEPT at least one + * record so the rktp materialises on rkt_desp (precondition + * for the metadata-error propagation path to fire). */ + while (consumed < 1 && attempts++ < 30) { + rd_kafka_messages_destroy(batch); + batch = NULL; + error = rd_kafka_share_poll(consumer, 1000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + rd_kafka_share_acknowledge(consumer, msg); + consumed++; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(consumed >= 1, + "Pre-condition: expected to consume + ack at least one " + "record before deleting the topic"); + + TEST_SAY("Deleting topic %s\n", topic); + topics_for_delete[0] = topic_dup; + del_err = test_DeleteTopics_simple(common_admin, NULL, + topics_for_delete, 1, NULL); + TEST_ASSERT(!del_err, "DeleteTopics failed: %s", + rd_kafka_err2str(del_err)); + + /* Wait for the topic delete to propagate in the cluster. */ + rd_sleep(3); + + /* Drain consume_batch and verify no topic-level error reaches + * the application. _STATE / generic timeouts are expected + * (no more records); only the metadata-derived topic codes + * must not surface. */ + for (post_attempts = 0; post_attempts < 30; post_attempts++) { + rd_kafka_messages_destroy(batch); + batch = NULL; + error = rd_kafka_share_poll(consumer, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + if (error) { + rd_kafka_resp_err_t code = rd_kafka_error_code(error); + TEST_SAY("Post-delete consume_batch: %s\n", + rd_kafka_err2name(code)); + if (code == RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION || + code == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART || + code == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC || + code == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || + code == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID || + code == + RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED) + surfaced_topic_err = rd_true; + rd_kafka_error_destroy(error); + continue; + } + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) + rd_kafka_share_acknowledge(consumer, msg); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(!surfaced_topic_err, + "Topic-level metadata errors must not surface to " + "the application after the subscribed topic is " + "deleted (transient codes are log-only)"); + + rd_free(topic_dup); + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test subscribing to 15 topics - triggers multiple fetch responses + * + * Creates 15 topics, subscribes to all of them, produces messages to each, + * and verifies all messages are consumed. This tests the scenario where + * topics are spread across multiple brokers and require multiple + * ShareFetch responses. + */ +static void do_test_subscribe_15_topics(void) { + const char *group = test_mk_topic_name("share-15topics", 1); + const int topic_cnt = 15; + const int msgs_per_topic = 100; + const int total_expected = topic_cnt * msgs_per_topic; + char *topics[15]; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + int consumed = 0; + int attempts; + int t; + + SUB_TEST("subscribe-15-topics"); + + /* Create 15 topics */ + for (t = 0; t < topic_cnt; t++) { + topics[t] = rd_strdup(test_mk_topic_name("0170-15topics", 1)); + test_create_topic_wait_exists(NULL, topics[t], 1, -1, 30000); + } + + /* Produce messages to each topic */ + for (t = 0; t < topic_cnt; t++) { + test_produce_msgs_simple(common_producer, topics[t], 0, + msgs_per_topic); + } + TEST_SAY("Produced %d messages to %d topics\n", total_expected, + topic_cnt); + + /* Create consumer */ + rkshare = test_create_share_consumer(group, NULL); + + /* Set group offset */ + test_share_set_auto_offset_reset(group, "earliest"); + + /* Subscribe to all topics */ + subs = rd_kafka_topic_partition_list_new(topic_cnt); + for (t = 0; t < topic_cnt; t++) { + rd_kafka_topic_partition_list_add(subs, topics[t], + RD_KAFKA_PARTITION_UA); + } + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("Subscribed to %d topics\n", topic_cnt); + + /* Consume all messages */ + attempts = 100; + while (consumed < total_expected && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(rkshare, 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + + if (rcvd > 0) + TEST_SAY("Progress: %d/%d\n", consumed, total_expected); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == total_expected, + "Expected %d messages, consumed %d", total_expected, + consumed); + + /* Cleanup */ + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + for (t = 0; t < topic_cnt; t++) { + rd_free(topics[t]); + } + + SUB_TEST_PASS(); +} + + +/** + * @brief Test share.auto.offset.reset = earliest + * + * Verifies that with share.auto.offset.reset = earliest, consumer receives + * all messages including those produced before subscription. + */ +static void do_test_auto_offset_reset_earliest(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-offset-earliest-test"; + rd_kafka_topic_partition_list_t *subs; + int consumed = 0; + int attempts; + const int msg_cnt = 100; + + SUB_TEST("share.auto.offset.reset=earliest"); + + /* Create topic */ + topic = test_mk_topic_name("0170-offset-earliest", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Produce messages BEFORE consumer subscribes */ + TEST_SAY("Producing %d messages BEFORE subscription...\n", msg_cnt); + test_produce_msgs_easy(topic, 0, 0, msg_cnt); + + /* Create consumer */ + consumer = test_create_share_consumer(group, NULL); + + /* Configure group with earliest offset */ + test_share_set_auto_offset_reset(group, "earliest"); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume - should get all 100 messages */ + TEST_SAY("Consuming with earliest offset...\n"); + attempts = 50; + while (consumed < msg_cnt && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + + if (rcvd > 0) + TEST_SAY("Progress: %d/%d\n", consumed, msg_cnt); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == msg_cnt, "Expected %d messages, got %d", + msg_cnt, consumed); + + TEST_SAY("SUCCESS: earliest offset - consumed all %d messages\n", + consumed); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * @brief Test share.auto.offset.reset default value (latest) + * + * Verifies that with default (latest) offset, consumer only receives + * messages produced after subscription, not pre-existing messages. + */ +static void do_test_auto_offset_reset_default_latest(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-offset-default-test"; + rd_kafka_topic_partition_list_t *subs; + int consumed_before = 0, consumed_after = 0; + int attempts; + const int initial_msgs = 100; + const int later_msgs = 50; + + SUB_TEST("share.auto.offset.reset=latest (default)"); + + /* Create topic */ + topic = test_mk_topic_name("0170-offset-default", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Produce messages BEFORE consumer subscribes */ + TEST_SAY("Producing %d messages BEFORE subscription...\n", + initial_msgs); + test_produce_msgs_easy(topic, 0, 0, initial_msgs); + + /* Create consumer - NO offset reset config (uses default "latest") */ + consumer = test_create_share_consumer(group, NULL); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Allow consumer to establish position */ + rd_sleep(3); + + /* Try to consume - should get 0 messages (default is latest) */ + TEST_SAY("Trying to consume with default (latest) offset...\n"); + attempts = 10; + while (attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed_before++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY( + "Consumed %d messages from before subscription (expected: 0)\n", + consumed_before); + TEST_ASSERT(consumed_before == 0, + "With default (latest) offset, should not receive " + "pre-existing messages, got %d", + consumed_before); + + /* Now produce more messages */ + TEST_SAY("Producing %d more messages AFTER subscription...\n", + later_msgs); + test_produce_msgs_easy(topic, 0, 0, later_msgs); + + /* Consume - should get the new 50 messages */ + TEST_SAY("Consuming messages produced after subscription...\n"); + attempts = 30; + while (consumed_after < later_msgs && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed_after++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumed %d messages after subscription (expected: %d)\n", + consumed_after, later_msgs); + TEST_ASSERT(consumed_after == later_msgs, + "Expected %d messages after subscription, got %d", + later_msgs, consumed_after); + + TEST_SAY( + "SUCCESS: default (latest) offset verified - before=%d, " + "after=%d\n", + consumed_before, consumed_after); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * @brief rd_kafka_share_subscribe input validation against a real + * broker. + * + * Covers the share-subscribe API surface contract: + * 1. A topic name starting with '^' is treated as a literal — no + * regex compilation, no rejection. The name is forwarded + * verbatim to the broker via + * ShareGroupHeartbeat.SubscribedTopicNames; the broker silently + * drops names it cannot resolve. + * 2. Subscription list containing an empty topic name returns + * INVALID_ARG. + * 3. Duplicate topic names return INVALID_ARG. + */ +static void do_test_subscribe_input_validation(void) { + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + const char *group = "share-subscribe-validation"; + + SUB_TEST(); + + /* Case 1: '^foo.*' is forwarded as a literal, not interpreted + * as a regex. */ + consumer = test_create_share_consumer(group, NULL); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, "^foo.*", + RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(consumer, subs); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "expected NO_ERROR for '^foo.*' literal topic, got: %s", + rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subs); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Case 2: entry with empty topic name. */ + consumer = test_create_share_consumer(group, NULL); + + subs = rd_kafka_topic_partition_list_new(2); + rd_kafka_topic_partition_list_add(subs, "valid-topic", + RD_KAFKA_PARTITION_UA); + rd_kafka_topic_partition_list_add(subs, "", RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(consumer, subs); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "expected INVALID_ARG for empty topic name, got: %s", + rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subs); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Case 3: duplicate topic names. */ + consumer = test_create_share_consumer(group, NULL); + + subs = rd_kafka_topic_partition_list_new(2); + rd_kafka_topic_partition_list_add(subs, "duplicate-topic", + RD_KAFKA_PARTITION_UA); + rd_kafka_topic_partition_list_add(subs, "duplicate-topic", + RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(consumer, subs); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "expected INVALID_ARG for duplicate topic, got: %s", + rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subs); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * @brief End-to-end verification that '^'-prefixed entries are + * forwarded as literals (no regex resolution). + * + * Creates a real topic, produces records to it, then: + * Phase 1. Subscribes with a regex-shaped string '^$' that + * would match the topic if treated as a regex. The broker + * cannot resolve the literal name and silently drops it + * from the assignment. The share consumer must receive + * zero records. + * Phase 2. Unsubscribes, resubscribes with the plain topic name. + * The consumer must now receive all produced records. + */ +static void do_test_subscribe_caret_treated_as_literal_e2e(void) { + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-caret-literal-e2e"; + const int msg_cnt = 50; + char caret_pattern[512]; + size_t records_phase1 = 0; + size_t records_phase2 = 0; + int attempts; + + SUB_TEST(); + + topic = test_mk_topic_name("0170-caret-literal", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_share_set_auto_offset_reset(group, "earliest"); + test_produce_msgs_easy(topic, 0, 0, msg_cnt); + + consumer = test_create_share_consumer(group, NULL); + + /* Phase 1: '^$' as a literal — broker drops, no records. */ + rd_snprintf(caret_pattern, sizeof(caret_pattern), "^%s$", topic); + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, caret_pattern, + RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(consumer, subs)); + rd_kafka_topic_partition_list_destroy(subs); + + /* rd_kafka_share_subscription must return the subscribed names + * verbatim (the '^' prefix is not stripped). */ + subs = NULL; + TEST_CALL_ERR__(rd_kafka_share_subscription(consumer, &subs)); + TEST_ASSERT(subs && subs->cnt == 1, + "Phase 1: expected subscription cnt 1, got %d", + subs ? subs->cnt : -1); + TEST_ASSERT(!strcmp(subs->elems[0].topic, caret_pattern), + "Phase 1: expected subscription[0]='%s', got '%s'", + caret_pattern, subs->elems[0].topic); + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("Phase 1: subscribed with '%s' (literal); polling...\n", + caret_pattern); + attempts = 10; + while (attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) + rd_kafka_error_destroy(err); + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + records_phase1++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(records_phase1 == 0, + "Phase 1: expected 0 records for '%s' literal " + "subscription, got %zu", + caret_pattern, records_phase1); + TEST_SAY("Phase 1: received 0 records as expected\n"); + + /* Phase 2: resubscribe with the plain topic name — broker + * resolves it, consumer receives records. */ + TEST_CALL_ERR__(rd_kafka_share_unsubscribe(consumer)); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(consumer, subs)); + rd_kafka_topic_partition_list_destroy(subs); + + /* rd_kafka_share_subscription must now return the plain name. */ + subs = NULL; + TEST_CALL_ERR__(rd_kafka_share_subscription(consumer, &subs)); + TEST_ASSERT(subs && subs->cnt == 1, + "Phase 2: expected subscription cnt 1, got %d", + subs ? subs->cnt : -1); + TEST_ASSERT(!strcmp(subs->elems[0].topic, topic), + "Phase 2: expected subscription[0]='%s', got '%s'", topic, + subs->elems[0].topic); + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("Phase 2: resubscribed with plain name '%s'; consuming...\n", + topic); + attempts = 10; + while (records_phase2 < (size_t)msg_cnt && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 2000, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) + rd_kafka_error_destroy(err); + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + records_phase2++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(records_phase2 == (size_t)msg_cnt, + "Phase 2: expected %d records, got %zu", msg_cnt, + records_phase2); + TEST_SAY("Phase 2: received %zu/%d records\n", records_phase2, msg_cnt); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * @brief Verify consume_batch surfaces RD_KAFKA_RESP_ERR__STATE with + * "not subscribed" when there is no active subscription. + * + * Walks the subscription state machine: + * Case 1: never subscribed -> __STATE + * Case 2: after subscribe -> not __STATE (recovery) + * Case 3: after unsubscribe -> __STATE + * Case 4: after re-subscribe following unsub -> not __STATE (recovery) + * Case 5: subscribe([]) (== unsubscribe) -> __STATE + */ +static void do_test_consume_batch_without_subscription(void) { + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + size_t rcvd = 0; + const char *group = "share-no-subscription"; + const char *topic; + + SUB_TEST(); + + topic = test_mk_topic_name("0170-no-subscription", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + consumer = test_create_share_consumer(group, NULL); + + /* Case 1: never subscribed. */ + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(err != NULL, + "Case 1 (never subscribed): expected error, got NULL"); + TEST_ASSERT(rd_kafka_error_code(err) == RD_KAFKA_RESP_ERR__STATE, + "Case 1: expected __STATE, got: %s", + rd_kafka_err2name(rd_kafka_error_code(err))); + TEST_ASSERT(strstr(rd_kafka_error_string(err), "not subscribed"), + "Case 1: expected 'not subscribed' substring, got: %s", + rd_kafka_error_string(err)); + TEST_ASSERT(rcvd == 0, "Case 1: expected 0 msgs, got %zu", rcvd); + rd_kafka_error_destroy(err); + + /* Case 2: subscribed -> must not surface __STATE. */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(consumer, subs)); + rd_kafka_topic_partition_list_destroy(subs); + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) { + TEST_ASSERT(rd_kafka_error_code(err) != + RD_KAFKA_RESP_ERR__STATE, + "Case 2 (subscribed): unexpected __STATE: %s", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + } + + /* Case 3: unsubscribed -> __STATE again. */ + TEST_CALL_ERR__(rd_kafka_share_unsubscribe(consumer)); + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(err != NULL, + "Case 3 (after unsubscribe): expected error, got NULL"); + TEST_ASSERT(rd_kafka_error_code(err) == RD_KAFKA_RESP_ERR__STATE, + "Case 3: expected __STATE, got: %s", + rd_kafka_err2name(rd_kafka_error_code(err))); + TEST_ASSERT(strstr(rd_kafka_error_string(err), "not subscribed"), + "Case 3: expected 'not subscribed' substring, got: %s", + rd_kafka_error_string(err)); + TEST_ASSERT(rcvd == 0, "Case 3: expected 0 msgs, got %zu", rcvd); + rd_kafka_error_destroy(err); + + /* Case 4: re-subscribed after unsubscribe -> must not surface __STATE. + */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(consumer, subs)); + rd_kafka_topic_partition_list_destroy(subs); + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + if (err) { + TEST_ASSERT(rd_kafka_error_code(err) != + RD_KAFKA_RESP_ERR__STATE, + "Case 4 (re-subscribed): unexpected __STATE: %s", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + } + + /* Case 5: subscribe([]) is equivalent to unsubscribe — must + * return NO_ERROR and clear the F_SUBSCRIPTION flag, so the + * next consume_batch returns __STATE. */ + rd_kafka_resp_err_t empty_err; + + subs = rd_kafka_topic_partition_list_new(0); + empty_err = rd_kafka_share_subscribe(consumer, subs); + TEST_ASSERT(empty_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Case 5 (subscribe([])): expected NO_ERROR " + "(treated as unsubscribe), got: %s", + rd_kafka_err2name(empty_err)); + rd_kafka_topic_partition_list_destroy(subs); + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(err != NULL, + "Case 5 (after subscribe([])): expected error, got NULL"); + TEST_ASSERT(rd_kafka_error_code(err) == RD_KAFKA_RESP_ERR__STATE, + "Case 5: expected __STATE, got: %s", + rd_kafka_err2name(rd_kafka_error_code(err))); + TEST_ASSERT(strstr(rd_kafka_error_string(err), "not subscribed"), + "Case 5: expected 'not subscribed' substring, got: %s", + rd_kafka_error_string(err)); + TEST_ASSERT(rcvd == 0, "Case 5: expected 0 msgs, got %zu", rcvd); + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/* Extends test_ack_cb_state_t to also record the topic of the first + * callback invocation, so do_test_subscription_change_acks_pending can + * verify that the implicit ack driven by the subscription change was for + * t1 (not later t2 acks fired by subsequent polls in the loop). */ +typedef struct subchg_ack_state_s { + test_ack_cb_state_t base; /* must be first - cast-compat with base */ + char first_callback_topic[256]; +} subchg_ack_state_t; + +static void subchg_ack_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + subchg_ack_state_t *st = (subchg_ack_state_t *)opaque; + const rd_kafka_share_partition_offsets_t *entry; + const rd_kafka_topic_partition_t *part; + + (void)rkshare; + + /* Mirror test_share_ack_cb's base accounting */ + test_ack_cb_state_push_err(&st->base, err); + + entry = rd_kafka_share_partition_offsets_list_get(partitions, 0); + if (!entry) + return; + + st->base.total_offsets += + rd_kafka_share_partition_offsets_offsets_cnt(entry); + + /* Record the topic of the first callback invocation only. */ + if (st->base.callback_cnt == 1) { + part = rd_kafka_share_partition_offsets_partition(entry); + if (part && part->topic) + rd_snprintf(st->first_callback_topic, + sizeof(st->first_callback_topic), "%s", + part->topic); + } +} + + +/** + * @brief Test that a subscription change drives an implicit ack of the + * previously-fetched batch, firing the share ack callback. + * + * Produce one record to t1 and one to t2. In implicit mode, subscribe to t1, + * fetch the t1 record (without explicitly acknowledging), then re-subscribe + * to t2. The subscription change should cause t1's outstanding record to be + * implicitly acknowledged, firing the ack commit callback for t1's + * partition. + * + * A custom callback records the topic of the first callback invocation so + * we can verify the callback fired for t1 (not for t2's implicit ack, + * which can also fire if the polling loop iterates past the t2 record's + * arrival). + */ +static void do_test_subscription_change_acks_pending(void) { + char *group; + char *t1; + char *t2; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + subchg_ack_state_t state = {0}; + size_t rcvd = 0; + int attempts; + + SUB_TEST(); + + /* rd_strdup each test_mk_topic_name() result because the helper + * returns a pointer to a single TLS static buffer that gets + * clobbered by the next call. */ + group = rd_strdup(test_mk_topic_name("share-sub-change-ack", 1)); + t1 = rd_strdup(test_mk_topic_name("0170-subchg-t1", 1)); + t2 = rd_strdup(test_mk_topic_name("0170-subchg-t2", 1)); + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== subscription-change-drives-ack-callback ===\n"); + TEST_SAY( + "============================================================\n"); + + test_create_topic_wait_exists(NULL, t1, 1, -1, 30000); + test_create_topic_wait_exists(NULL, t2, 1, -1, 30000); + + test_produce_msgs_simple(common_producer, t1, 0, 1); + test_produce_msgs_simple(common_producer, t2, 0, 1); + + rkshare = test_create_share_consumer_with_cb( + group, "implicit", &state.base, subchg_ack_cb); + test_share_set_auto_offset_reset(group, "earliest"); + + /* Subscribe to t1 only */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, t1, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Poll until the t1 record arrives */ + attempts = 30; + while (rcvd < 1 && attempts-- > 0) { + size_t batch_rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(rkshare, 2000, &batch); + batch_rcvd = rd_kafka_messages_count(batch); + if (err) + rd_kafka_error_destroy(err); + rcvd += batch_rcvd; + } + TEST_ASSERT(rcvd == 1, "Expected 1 record from t1, got %zu", rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(state.base.callback_cnt == 0, + "Did not expect callback before subscription change, " + "got %d", + state.base.callback_cnt); + + /* Re-subscribe to t2 only - this should drive an implicit ack of + * t1's outstanding batch. */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, t2, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Poll a few times to drive the callback and pick up the t2 record. + * Subsequent polls in this loop may also piggyback t2's implicit + * ack and produce a second callback, but the FIRST callback must + * be the one driven by the t1 -> t2 subscription change. The + * custom callback records the topic of the first invocation so + * we can assert that below. */ + rcvd = 0; + attempts = 30; + while ((rcvd < 1 || state.base.callback_cnt < 1) && attempts-- > 0) { + size_t batch_rcvd = 0; + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(rkshare, 1000, &batch); + batch_rcvd = rd_kafka_messages_count(batch); + if (err) + rd_kafka_error_destroy(err); + rcvd += batch_rcvd; + } + + TEST_ASSERT(rcvd == 1, "Expected 1 record from t2, got %zu", rcvd); + TEST_ASSERT(state.base.callback_cnt >= 1, + "Expected ack callback to fire after subscription change, " + "got %d callbacks", + state.base.callback_cnt); + TEST_ASSERT( + test_ack_cb_state_first_err(&state.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected no error in callback, got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + TEST_ASSERT(strcmp(state.first_callback_topic, t1) == 0, + "Expected first ack callback to be for t1 (%s), got %s", t1, + state.first_callback_topic); + + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + rd_free(t1); + rd_free(t2); + rd_free(group); + + TEST_SAY("=== subscription-change-drives-ack-callback: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test two share groups on the same topic with different + * auto.offset.reset policies (earliest vs latest). + * + * Group A is configured for earliest, group B for latest. Records produced + * before either group joins are visible only to group A; records produced + * after both have joined are visible to both. + */ +static void do_test_two_groups_earliest_vs_latest(void) { + char *grpA; + char *grpB; + char *topic; + rd_kafka_share_t *consA, *consB; + rd_kafka_topic_partition_list_t *subs; + int countA = 0, countB = 0; + int attempts; + + SUB_TEST(); + + /* rd_strdup each test_mk_topic_name() result because the helper + * returns a pointer to a single TLS static buffer that gets + * clobbered by the next call. Without this, grpA, grpB and topic + * would all alias to the same buffer. */ + grpA = rd_strdup(test_mk_topic_name("share-grpA-earliest", 1)); + grpB = rd_strdup(test_mk_topic_name("share-grpB-latest", 1)); + topic = rd_strdup(test_mk_topic_name("0170-evl", 1)); + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== two-groups-earliest-vs-latest ===\n"); + TEST_SAY( + "============================================================\n"); + + test_create_topic_wait_exists(NULL, topic, 1, -1, 30000); + + /* Produce 5 records BEFORE the consumers join. These are the + * "original records present in the broker" that earliest fetches + * and latest skips. */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Match the working librdkafka ordering used by + * do_test_multi_consumer_overlap (0170) and 0171's + * test_poll_callback_piggybacked_acks: + * produce -> create consumer -> set reset -> subscribe -> poll. + * Setting share.auto.offset.reset before any consumer exists at + * the broker doesn't take effect; setting it after consumer + * creation but before subscribe does. */ + consA = test_create_share_consumer(grpA, NULL); + consB = test_create_share_consumer(grpB, NULL); + + test_share_set_auto_offset_reset(grpA, "earliest"); + test_share_set_auto_offset_reset(grpB, "latest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consA, subs); + rd_kafka_share_subscribe(consB, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* groupA (earliest) fetches the 5 pre-existing records; + * groupB (latest) skips them (HW=5 anchors the start). */ + attempts = 30; + while (attempts-- > 0 && countA < 5) { + int batch_cnt = 0; + test_share_poll(consA, 1000, NULL, 0, &batch_cnt); + countA += batch_cnt; + + batch_cnt = 0; + test_share_poll(consB, 500, NULL, 0, &batch_cnt); + countB += batch_cnt; + } + TEST_ASSERT(countA == 5, "groupA (earliest) expected 5 records, got %d", + countA); + TEST_ASSERT(countB == 0, + "groupB (latest) expected 0 records for pre-subscribe " + "produce, got %d", + countB); + + /* Produce 3 more records AFTER both groups have joined - both + * groups should see these (earliest continues from HW=5, latest + * advances from HW=5 too). */ + test_produce_msgs_simple(common_producer, topic, 0, 3); + + attempts = 30; + while (attempts-- > 0 && countB < 3) { + int batch_cnt = 0; + test_share_poll(consA, 500, NULL, 0, &batch_cnt); + countA += batch_cnt; + + batch_cnt = 0; + test_share_poll(consB, 1000, NULL, 0, &batch_cnt); + countB += batch_cnt; + } + TEST_ASSERT(countA == 8, "groupA expected 8 records total, got %d", + countA); + TEST_ASSERT(countB == 3, + "groupB expected 3 records (post-subscribe only), got %d", + countB); + + test_share_consumer_close(consA); + test_share_consumer_close(consB); + test_share_destroy(consA); + test_share_destroy(consB); + + rd_free(grpA); + rd_free(grpB); + rd_free(topic); + + TEST_SAY("=== two-groups-earliest-vs-latest: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that DeleteRecords advances the LSO and a new share group + * with auto.offset.reset=earliest starts at the new LSO. + * + * Produce 10 records, DeleteRecords up to offset 5 (LSO -> 5), then create + * a new share group with earliest and verify exactly 5 records + * (offsets 5..9) are consumed. + */ +static void do_test_delete_records_advances_lso(void) { + char *group; + char *topic; + rd_kafka_topic_partition_list_t *offsets; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err_obj; + rd_kafka_resp_err_t err; + int64_t first_offset = -1; + int consumed = 0; + size_t batch_cnt = 0; + int attempts; + size_t k; + + SUB_TEST(); + + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) + SUB_TEST_SKIP( + "delete-records scenario not supported (broker on " + "Windows)\n"); + + group = rd_strdup(test_mk_topic_name("share-lso-delete", 1)); + topic = rd_strdup(test_mk_topic_name("0170-lso", 1)); + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== delete-records-advances-lso ===\n"); + TEST_SAY( + "============================================================\n"); + + test_create_topic_wait_exists(NULL, topic, 1, -1, 30000); + test_produce_msgs_simple(common_producer, topic, 0, 10); + + /* DeleteRecords up to offset 5: deletes offsets 0..4, LSO moves to 5 */ + offsets = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(offsets, topic, 0)->offset = 5; + err = test_DeleteRecords_simple(common_admin, NULL, offsets, NULL); + rd_kafka_topic_partition_list_destroy(offsets); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "DeleteRecords failed: %s", rd_kafka_err2name(err)); + + /* New share group with earliest should start at the new LSO */ + test_share_set_auto_offset_reset(group, "earliest"); + rkshare = test_create_share_consumer(group, NULL); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 30; + while (consumed < 5 && attempts-- > 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + err_obj = rd_kafka_share_poll(rkshare, 2000, &batch); + batch_cnt = rd_kafka_messages_count(batch); + if (err_obj) { + rd_kafka_error_destroy(err_obj); + continue; + } + for (k = 0; k < batch_cnt; k++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, k); + if (!msg->err) { + if (first_offset == -1) + first_offset = msg->offset; + consumed++; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(consumed == 5, + "Expected 5 records (offsets 5..9) after DeleteRecords, " + "consumed %d", + consumed); + TEST_ASSERT(first_offset == 5, + "Expected first delivered offset to be 5 (new LSO after " + "DeleteRecords), got %" PRId64, + first_offset); + + /* Verify no additional records show up */ + rd_kafka_messages_destroy(batch); + batch = NULL; + err_obj = rd_kafka_share_poll(rkshare, 1000, &batch); + batch_cnt = rd_kafka_messages_count(batch); + if (err_obj) + rd_kafka_error_destroy(err_obj); + TEST_ASSERT(batch_cnt == 0, + "Did not expect more records after consuming 5, got %zu", + batch_cnt); + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + rd_free(group); + rd_free(topic); + + TEST_SAY("=== delete-records-advances-lso: PASSED ===\n"); + + SUB_TEST_PASS(); +} + +/* subscribe([]) is equivalent to unsubscribe — must clear the + * subscription flag so the subsequent poll surfaces __STATE. */ +static const test_scenario_t test_poll_after_empty_subscribe = { + .name = "poll-after-empty-subscribe", + .ops = {SUBSCRIBE(1), PRODUCE(5), CONSUME_ANY(), + SUBSCRIBE_EMPTY(), /* Empty list == unsubscribe */ + POLL_NO_SUB(), TEST_OPS_END()}}; + +int main_0170_share_consumer_subscription(int argc, char **argv) { + /* Create common handles for all tests */ + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + test_timeout_set(200); + + /* Auto offset reset tests */ + do_test_auto_offset_reset_earliest(); + do_test_auto_offset_reset_default_latest(); + + /* Basic subscription tests */ + do_test_scenario(&test_single_subscribe); + do_test_scenario(&test_single_unsubscribe); + do_test_scenario(&test_repeated_subscribe); + do_test_scenario(&test_repeated_unsubscribe); + + /* Subscription replacement tests */ + /* TODO KIP-932: test_topic_switch might be incorrect. Verify and + * remove. */ + // do_test_scenario(&test_topic_switch); + do_test_scenario(&test_incremental_subscription); + + /* Edge case tests */ + do_test_scenario(&test_subscribe_before_topic_exists); + do_test_scenario(&test_poll_empty_topic); + do_test_scenario(&test_poll_no_subscription); + do_test_scenario(&test_poll_after_unsubscribe); + + /* Topic deletion tests (Skipped for Windows)*/ + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) + TEST_SAY( + "Skipping topic deletion scenario" + "(broker on Windows)\n"); + else + do_test_scenario(&test_topic_deletion); + + /* Verify that topic deletion does not surface a topic-level + * error code to consume_batch (transient codes are log-only). */ + test_topic_deletion_does_not_surface_error(); + + /* Stress tests */ + do_test_scenario(&test_rapid_updates); + + /* Multi-consumer tests (standalone - requires shared topics) */ + do_test_multi_consumer_overlap(); + + /* Scale tests (many topics) */ + do_test_subscribe_15_topics(); + + /* Subscription change drives ack callback */ + do_test_subscription_change_acks_pending(); + + /* Two groups with earliest vs latest offset reset */ + do_test_two_groups_earliest_vs_latest(); + + /* DeleteRecords advances LSO; new group with earliest sees fewer + * records (skipped on Windows brokers). */ + do_test_delete_records_advances_lso(); + + /* Input validation tests */ + do_test_subscribe_input_validation(); + do_test_subscribe_caret_treated_as_literal_e2e(); + + /* Subscription state guard tests */ + do_test_consume_batch_without_subscription(); + do_test_scenario(&test_poll_after_empty_subscribe); + + /* Cleanup common handles */ + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0171-share_consumer_consume.c b/lib/librdkafka-2.15.0/tests/0171-share_consumer_consume.c new file mode 100644 index 00000000000..ad53fd5a01d --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0171-share_consumer_consume.c @@ -0,0 +1,2150 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +/** + * @brief Maximum supported values for test configuration + */ +#define MAX_CONSUMERS 16 +#define MAX_TOPICS 16 +#define MAX_PARTITIONS 32 +#define BATCH_SIZE 10000 + +/** Common producer reused across all tests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all tests. */ +static rd_kafka_t *common_admin; + +/** + * @brief Test configuration structure + * + * This structure defines all parameters for a share consumer test scenario. + * + * Example configurations: + * + * 1. Multiple consumers, single topic, multiple partitions: + * { .consumer_cnt = 2, .topic_cnt = 1, .partitions = {3}, + * .msgs_per_partition = 100, .group_name = "test-group" } + * + * 2. Single consumer, multiple topics, single partition each: + * { .consumer_cnt = 1, .topic_cnt = 2, .partitions = {1, 1}, + * .msgs_per_partition = 100, .group_name = "test-group" } + * + * 3. Multiple consumers, multiple topics, multiple partitions: + * { .consumer_cnt = 3, .topic_cnt = 2, .partitions = {3, 2}, + * .msgs_per_partition = 50, .group_name = "test-group" } + */ +typedef struct { + int consumer_cnt; /**< Number of consumers to create */ + int topic_cnt; /**< Number of topics to create */ + int partitions[MAX_TOPICS]; /**< Partition count for each topic */ + int msgs_per_partition; /**< Messages to produce per partition */ + const char *group_name; /**< Share group name */ + const char *test_name; /**< Test description for logging */ + int poll_timeout_ms; /**< Timeout for each poll (default: 3000) */ + int max_attempts; /**< Max poll attempts (default: auto-calculated) */ +} share_test_config_t; + +/** + * @brief Test state/results structure + */ +typedef struct { + rd_kafka_share_t *consumers[MAX_CONSUMERS]; + char *topic_names[MAX_TOPICS]; + int per_consumer_count[MAX_CONSUMERS]; + int total_consumed; + int total_expected; +} share_test_state_t; + +/** + * @brief Create share consumers + */ +static void create_share_consumers(share_test_config_t *config, + share_test_state_t *state) { + int i; + + for (i = 0; i < config->consumer_cnt; i++) { + state->consumers[i] = + test_create_share_consumer(config->group_name, NULL); + } + + TEST_SAY("Created %d share consumer(s)\n", config->consumer_cnt); +} + +/** + * @brief Create topics and produce messages + */ +static void setup_topics_and_produce(share_test_config_t *config, + share_test_state_t *state) { + int t, p; + int total_partitions = 0; + + state->total_expected = 0; + + for (t = 0; t < config->topic_cnt; t++) { + /* Generate unique topic name */ + state->topic_names[t] = + rd_strdup(test_mk_topic_name("0171-share-test", 1)); + + /* Create topic with specified partitions */ + test_create_topic_wait_exists(NULL, state->topic_names[t], + config->partitions[t], -1, + 60 * 1000); + + /* Produce messages to each partition */ + for (p = 0; p < config->partitions[t]; p++) { + test_produce_msgs_simple(common_producer, + state->topic_names[t], p, + config->msgs_per_partition); + state->total_expected += config->msgs_per_partition; + } + + total_partitions += config->partitions[t]; + TEST_SAY("Topic '%s': %d partition(s), %d msgs/partition\n", + state->topic_names[t], config->partitions[t], + config->msgs_per_partition); + } + + TEST_SAY( + "Setup complete: %d topic(s), %d total partition(s), " + "%d total messages\n", + config->topic_cnt, total_partitions, state->total_expected); +} + +/** + * @brief Subscribe consumers to topics + */ +static void subscribe_consumers(share_test_config_t *config, + share_test_state_t *state) { + rd_kafka_topic_partition_list_t *subs; + int t, i; + + /* Set group config using a dedicated admin client */ + test_share_set_auto_offset_reset(config->group_name, "earliest"); + + /* Build subscription list */ + subs = rd_kafka_topic_partition_list_new(config->topic_cnt); + for (t = 0; t < config->topic_cnt; t++) { + rd_kafka_topic_partition_list_add(subs, state->topic_names[t], + RD_KAFKA_PARTITION_UA); + } + + /* Subscribe all consumers */ + for (i = 0; i < config->consumer_cnt; i++) { + rd_kafka_share_subscribe(state->consumers[i], subs); + } + + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("Subscribed %d consumer(s) to %d topic(s)\n", + config->consumer_cnt, config->topic_cnt); +} + +/** + * @brief Consume messages function + */ +static void consume_messages(share_test_config_t *config, + share_test_state_t *state) { + rd_kafka_messages_t *batch = NULL; + int attempts; + int poll_timeout; + int i; + + /* Initialize per-consumer counts */ + for (i = 0; i < config->consumer_cnt; i++) { + state->per_consumer_count[i] = 0; + } + state->total_consumed = 0; + + /* Calculate defaults if not specified */ + poll_timeout = + config->poll_timeout_ms > 0 ? config->poll_timeout_ms : 3000; + + /* Auto-calculate attempts based on expected messages and consumers */ + if (config->max_attempts > 0) { + attempts = config->max_attempts; + } else { + /* Heuristic: at least 10 attempts, or based on message count */ + attempts = (state->total_expected / (BATCH_SIZE / 2)) + 10; + if (attempts < 20) + attempts = 20; + } + + TEST_SAY( + "Starting consumption: expecting %d messages, " + "max %d attempts, %dms timeout\n", + state->total_expected, attempts, poll_timeout); + + while (state->total_consumed < state->total_expected && + attempts-- > 0) { + /* Round-robin poll across all consumers */ + for (i = 0; i < config->consumer_cnt; i++) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(state->consumers[i], + poll_timeout, &batch); + + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) { + state->per_consumer_count[i]++; + state->total_consumed++; + } + } + + /* Early exit if we've consumed everything */ + if (state->total_consumed >= state->total_expected) + break; + } + + /* Progress logging */ + if (config->consumer_cnt == 1) { + TEST_SAY("Progress: %d/%d\n", state->total_consumed, + state->total_expected); + } else { + /* Build distribution string for multiple consumers */ + char dist[256] = {0}; + int pos = 0; + for (i = 0; i < config->consumer_cnt && pos < 250; + i++) { + pos += rd_snprintf( + dist + pos, sizeof(dist) - pos, "c%d=%d ", + i, state->per_consumer_count[i]); + } + TEST_SAY("Progress: %d/%d (%s)\n", + state->total_consumed, state->total_expected, + dist); + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; +} + +/** + * @brief Cleanup function + */ +static void cleanup_test(share_test_config_t *config, + share_test_state_t *state) { + int t, i; + + for (t = 0; t < config->topic_cnt; t++) { + if (state->topic_names[t]) { + rd_free(state->topic_names[t]); + state->topic_names[t] = NULL; + } + } + + /* Destroy consumers */ + for (i = 0; i < config->consumer_cnt; i++) { + if (state->consumers[i]) { + test_share_consumer_close(state->consumers[i]); + test_share_destroy(state->consumers[i]); + state->consumers[i] = NULL; + } + } + + TEST_SAY("Cleanup complete\n"); +} + +/** + * @brief Test Runner function + * + * This function handles: + * 1. Create consumers + * 2. Create topics and produce messages + * 3. Subscribe consumers + * 4. Consume and verify messages + * 5. Cleanup + * + * @param config Test configuration + * @returns 0 on success, -1 on failure + */ +static int run_share_consumer_test(share_test_config_t *config) { + share_test_state_t state = {0}; + int i; + char dist_str[512] = {0}; + int pos = 0; + char unique_suffix[64]; + char unique_group[128]; + char unique_test_name[256]; + + /* Validate config */ + TEST_ASSERT(config->consumer_cnt > 0 && + config->consumer_cnt <= MAX_CONSUMERS, + "consumer_cnt must be 1-%d", MAX_CONSUMERS); + TEST_ASSERT(config->topic_cnt > 0 && config->topic_cnt <= MAX_TOPICS, + "topic_cnt must be 1-%d", MAX_TOPICS); + TEST_ASSERT(config->group_name != NULL, "group_name is required"); + + /* Append a per-invocation unique suffix to the group name and + * displayed test name so re-runs / parallel runs on the same + * cluster can't collide on share-group state. */ + rd_snprintf(unique_suffix, sizeof(unique_suffix), "rnd%" PRIx64, + test_id_generate()); + rd_snprintf(unique_group, sizeof(unique_group), "%s-%s", + config->group_name, unique_suffix); + rd_snprintf(unique_test_name, sizeof(unique_test_name), "%s [%s]", + config->test_name ? config->test_name + : "Share Consumer Test", + unique_suffix); + config->group_name = unique_group; + config->test_name = unique_test_name; + + /* Print test header */ + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== %s ===\n", config->test_name); + TEST_SAY("=== Consumers: %d, Topics: %d, Partitions: [", + config->consumer_cnt, config->topic_cnt); + for (i = 0; i < config->topic_cnt; i++) { + TEST_SAY("%d%s", config->partitions[i], + i < config->topic_cnt - 1 ? ", " : ""); + } + TEST_SAY("], Msgs/Partition: %d ===\n", config->msgs_per_partition); + TEST_SAY( + "============================================================\n"); + + /* Execute test phases */ + create_share_consumers(config, &state); + setup_topics_and_produce(config, &state); + subscribe_consumers(config, &state); + consume_messages(config, &state); + + /* Verify results */ + TEST_ASSERT(state.total_consumed == state.total_expected, + "Expected %d messages, consumed %d", state.total_expected, + state.total_consumed); + + /* Build distribution string */ + for (i = 0; i < config->consumer_cnt && pos < 500; i++) { + pos += rd_snprintf(dist_str + pos, sizeof(dist_str) - pos, + "c%d=%d ", i, state.per_consumer_count[i]); + } + + TEST_SAY("✓ SUCCESS: Consumed %d/%d messages (distribution: %s)\n", + state.total_consumed, state.total_expected, dist_str); + + /* Cleanup */ + cleanup_test(config, &state); + + return 0; +} + + +/** + * @brief Single consumer, single topic, single partition + */ +static void do_test_single_consumer_single_topic_single_partition(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 1000, + .group_name = "share-1c-1t-1p", + .test_name = "Single consumer, single topic, single partition"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Single consumer, single topic, multiple partitions + */ +static void do_test_single_consumer_single_topic_multiple_partitions(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 1, + .partitions = {3}, + .msgs_per_partition = 500, + .group_name = "share-1c-1t-3p", + .test_name = "Single consumer, single topic, 3 partitions"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Single consumer, multiple topics, single partition each + */ +static void do_test_single_consumer_multiple_topic_single_partition(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 2, + .partitions = {1, 1}, + .msgs_per_partition = 500, + .group_name = "share-1c-2t-1p", + .test_name = "Single consumer, 2 topics, 1 partition each"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Single consumer, multiple topics, multiple partitions + */ +static void do_test_single_consumer_multiple_topic_multiple_partitions(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 3, + .partitions = {2, 2, 2}, + .msgs_per_partition = 500, + .group_name = "share-1c-3t-2p", + .test_name = "Single consumer, 3 topics, 2 partitions each"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Multiple consumers, single topic, single partition + */ +static void do_test_multiple_consumers_single_topic_single_partition(void) { + share_test_config_t config = { + .consumer_cnt = 2, + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 1000, + .group_name = "share-2c-1t-1p", + .test_name = "2 consumers, single topic, single partition"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Multiple consumers, single topic, multiple partitions + */ +static void do_test_multiple_consumers_single_topic_multiple_partitions(void) { + share_test_config_t config = { + .consumer_cnt = 2, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 500, + .group_name = "share-2c-1t-4p", + .test_name = "2 consumers, single topic, 4 partitions"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Multiple consumers, multiple topics, multiple partitions + */ +static void +do_test_multiple_consumers_multiple_topics_multiple_partitions(void) { + share_test_config_t config = { + .consumer_cnt = 3, + .topic_cnt = 2, + .partitions = {3, 3}, + .msgs_per_partition = 500, + .group_name = "share-3c-2t-3p", + .test_name = "3 consumers, 2 topics, 3 partitions each"}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/*************************************************************************** + * High Volume Tests + ***************************************************************************/ + +/** + * @brief High volume test with 10k messages on single partition + */ +static void do_test_high_volume_10k_messages(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 10000, + .group_name = "share-highvol-10k", + .test_name = "High volume: 10k messages, 1 partition", + .max_attempts = 100}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief High volume test with 50k messages across 5 partitions + */ +static void do_test_high_volume_50k_multi_partition(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 1, + .partitions = {5}, + .msgs_per_partition = 10000, + .group_name = "share-highvol-50k", + .test_name = "High volume: 50k messages, 5 partitions", + .max_attempts = 150}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Multi-Topic Tests (Triggers Multiple Fetch Responses) + ***************************************************************************/ + +/** + * @brief 15 topics to trigger multiple ShareFetch responses + */ +static void do_test_many_topics_15(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 15, + .partitions = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + .msgs_per_partition = 100, + .group_name = "share-15topics", + .test_name = "15 topics, 1 partition each (1500 msgs)", + .max_attempts = 100}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief 10 topics with 2 partitions each + */ +static void do_test_many_topics_10_multi_partition(void) { + share_test_config_t config = { + .consumer_cnt = 1, + .topic_cnt = 10, + .partitions = {2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + .msgs_per_partition = 100, + .group_name = "share-10topics-2p", + .test_name = "10 topics, 2 partitions each (2000 msgs)", + .max_attempts = 100}; + SUB_TEST(); + run_share_consumer_test(&config); + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Rapid Produce/Consume Tests + ***************************************************************************/ + +/** + * @brief Rapid produce/consume cycles - 20 rounds of 500 messages each + */ +static void do_test_rapid_produce_consume_cycles(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-rapid-cycles"; + rd_kafka_topic_partition_list_t *subs; + int round, total_consumed = 0; + const int rounds = 20; + const int msgs_per_round = 500; + const int total_expected = rounds * msgs_per_round; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== Rapid produce/consume cycles: %d rounds x %d msgs ===\n", + rounds, msgs_per_round); + + /* Create consumer and topic */ + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-rapid-cycles", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Configure group */ + test_share_set_auto_offset_reset(group, "earliest"); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Rapid cycles */ + for (round = 0; round < rounds; round++) { + int round_consumed = 0; + int attempts = 100; + + /* Produce */ + test_produce_msgs_simple(common_producer, topic, 0, + msgs_per_round); + + /* Consume */ + while (round_consumed < msgs_per_round && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + round_consumed++; + } + } + + total_consumed += round_consumed; + TEST_SAY("Round %d: consumed %d/%d (total: %d/%d)\n", round + 1, + round_consumed, msgs_per_round, total_consumed, + (round + 1) * msgs_per_round); + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(total_consumed == total_expected, + "Expected %d messages, consumed %d", total_expected, + total_consumed); + + TEST_SAY("SUCCESS: Rapid cycles completed - %d messages\n", + total_consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * @brief Empty topic then produce - verify consumer handles transition + */ +static void do_test_empty_then_produce(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-empty-then-produce"; + rd_kafka_topic_partition_list_t *subs; + int consumed = 0, attempts; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== Empty topic then produce test ===\n"); + + /* Create consumer and empty topic */ + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-empty-then-produce", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Configure and subscribe */ + test_share_set_auto_offset_reset(group, "earliest"); + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Poll empty topic - should return 0 */ + TEST_SAY("Polling empty topic...\n"); + for (attempts = 0; attempts < 5; attempts++) { + size_t rcvd = 0; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Empty poll %d: received %zu\n", attempts + 1, rcvd); + } + + /* Now produce messages */ + TEST_SAY("Producing 100 messages...\n"); + test_produce_msgs_simple(common_producer, topic, 0, 100); + + /* Consume - should get messages now */ + attempts = 100; + while (consumed < 100 && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == 100, "Expected 100 messages, consumed %d", + consumed); + TEST_SAY("SUCCESS: Empty then produce - consumed %d messages\n", + consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + +/** + * @brief Sparse partitions - produce only to some partitions + */ +static void do_test_sparse_partitions(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-sparse-partitions"; + rd_kafka_topic_partition_list_t *subs; + int consumed = 0, attempts; + const int msgs_per_partition = 100; + const int expected = 3 * msgs_per_partition; /* partitions 0,2,4 */ + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "=== Sparse partitions test (5 partitions, produce to 0,2,4) " + "===\n"); + + /* Create consumer and topic with 5 partitions */ + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-sparse-partitions", 1); + test_create_topic_wait_exists(NULL, topic, 5, -1, 60 * 1000); + + /* Configure and subscribe */ + test_share_set_auto_offset_reset(group, "earliest"); + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Produce only to partitions 0, 2, 4 (skip 1, 3) */ + test_produce_msgs_simple(common_producer, topic, 0, msgs_per_partition); + test_produce_msgs_simple(common_producer, topic, 2, msgs_per_partition); + test_produce_msgs_simple(common_producer, topic, 4, msgs_per_partition); + + TEST_SAY("Produced %d messages to partitions 0, 2, 4\n", expected); + + /* Consume all */ + attempts = 100; + while (consumed < expected && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + + TEST_SAY("Progress: %d/%d\n", consumed, expected); + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == expected, "Expected %d messages, consumed %d", + expected, consumed); + TEST_SAY("SUCCESS: Sparse partitions - consumed %d messages\n", + consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Poll callback - piggybacked acks on ShareFetch. + * + * In implicit mode, acks are automatically sent with the next poll(). + * This test verifies that the callback is invoked when acks are + * piggybacked on ShareFetch responses. + */ +static void do_test_poll_callback_piggybacked_acks(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + const char *topic; + const char *group = "share-poll-callback"; + rd_kafka_topic_partition_list_t *subs; + int consumed = 0, attempts; + test_ack_cb_state_t state = {0}; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "=== Poll callback test (piggybacked acks on ShareFetch) ===\n"); + + consumer = + test_create_share_consumer_with_cb(group, "implicit", &state, NULL); + + /* Create topic and produce messages */ + topic = test_mk_topic_name("0171-poll-cb", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 100); + + /* Configure group */ + test_share_set_auto_offset_reset(group, "earliest"); + + /* Subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* First poll: consume messages (implicit mode auto-acks) */ + attempts = 50; + while (consumed < 50 && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 2000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumed %d messages\n", consumed); + TEST_ASSERT(consumed > 0, "Expected to consume some messages"); + + /* Second poll: this sends piggybacked acks from first batch */ + test_wait_for_cb_with_poll(&state, consumer, 1, 20000); + + TEST_SAY("Callback count=%d, total_offsets=%zu\n", state.callback_cnt, + state.total_offsets); + + TEST_ASSERT( + state.callback_cnt >= 1, + "Expected at least 1 callback from piggybacked acks, got %d", + state.callback_cnt); + TEST_ASSERT(state.total_offsets == (size_t)consumed, + "Expected %d offsets in callback, got %zu", consumed, + state.total_offsets); + + TEST_SAY("SUCCESS: Poll callback received %d callbacks, %zu offsets\n", + state.callback_cnt, state.total_offsets); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + test_ack_cb_state_destroy(&state); + + SUB_TEST_PASS(); +} + + +/** + * @brief Worker-thread argument for the gate-conflict test. + * + * @c entering is set to 1 by the worker thread just before it calls + * share_consume_batch, so the main thread can wait for the worker to + * have started before probing. (entering=1 doesn't yet prove the gate + * is held — there's still a window between the atomic flip and the + * gate acquire inside consume_batch — so the main thread also retries + * its probe in a short polling loop below.) + */ +typedef struct { + rd_kafka_share_t *consumer; + rd_atomic32_t entering; +} blocker_arg_t; + +static int consume_blocker_thread(void *arg) { + blocker_arg_t *ba = arg; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + + rd_atomic32_set(&ba->entering, 1); + + /* Blocks for ~5s waiting for messages that never arrive. + * The share access gate is held for the full duration. */ + err = rd_kafka_share_poll(ba->consumer, 5000, &batch); + if (err) + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + return 0; +} + +/** + * @brief Verify the share consumer's single-thread access gate rejects + * concurrent calls from a second thread, and that ownership is + * released after the holder returns so a later call succeeds. + */ +static void do_test_concurrent_thread_access_rejected(void) { + rd_kafka_share_t *consumer; + const char *topic; + const char *group = "share-concurrent-thread"; + rd_kafka_topic_partition_list_t *subs; + thrd_t blocker; + blocker_arg_t ba; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err = NULL; + int probe_attempt; + const int probe_max_attempts = 50; /* 50 × 100ms = 5 s budget */ + + SUB_TEST(); + + /* Skip under Valgrind: Valgrind's heavy thread serialization + * makes the blocker thread's 5s consume_batch timeout + * unreliable, which can leave thrd_join / share consumer close + * waiting indefinitely (see logs.txt: subtest ran ~4000s + * before the suite watchdog killed it). */ + if (!strcmp(test_mode, "valgrind")) { + SUB_TEST_SKIP( + "Concurrent-thread gate test is scheduling-sensitive " + "under Valgrind\n"); + return; + } + + TEST_SAY("\n"); + TEST_SAY("=== Concurrent thread access rejected ===\n"); + + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-concurrent-thread", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Spawn the blocker. It enters consume_batch, takes the gate, + * and stays inside for ~5s on the empty topic. */ + ba.consumer = consumer; + rd_atomic32_init(&ba.entering, 0); + if (thrd_create(&blocker, consume_blocker_thread, &ba) != thrd_success) + TEST_FAIL("thrd_create failed"); + + /* Wait until the blocker thread has at least reached the point + * just before its consume_batch call. */ + while (rd_atomic32_get(&ba.entering) == 0) + rd_usleep(10 * 1000, NULL); + + /* The blocker has flipped its `entering` flag but may not yet + * have completed the gate acquire inside consume_batch. Probe + * in a short polling loop until we either observe __CONFLICT + * (gate is held by the blocker — the success case) or exhaust + * the budget. Without the loop, slow thread scheduling (e.g., + * Windows CI) lets the main thread win the gate first and the + * single-shot assertion flakes. */ + for (probe_attempt = 0; probe_attempt < probe_max_attempts; + probe_attempt++) { + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 0, &batch); + if (err && + rd_kafka_error_code(err) == RD_KAFKA_RESP_ERR__CONFLICT) + break; /* observed the conflict — done */ + if (err) { + rd_kafka_error_destroy(err); + err = NULL; + } + rd_usleep(100 * 1000, NULL); + } + + TEST_ASSERT( + err && rd_kafka_error_code(err) == RD_KAFKA_RESP_ERR__CONFLICT, + "Expected __CONFLICT from concurrent share_poll while blocker " + "thread holds the gate after %d probe attempts; got: %s", + probe_attempt + 1, + err ? rd_kafka_err2name(rd_kafka_error_code(err)) : "no error"); + rd_kafka_error_destroy(err); + + TEST_SAY("Gate correctly rejected concurrent caller\n"); + + /* Wait for blocker to return (releases the gate). */ + thrd_join(blocker, NULL); + + /* Sequential ownership transfer: gate is now free, the same + * main thread should be able to call share_poll without + * hitting the gate. */ + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 0, &batch); + TEST_ASSERT(!err, "After blocker returned, gate should be free; got %s", + err ? rd_kafka_err2str(rd_kafka_error_code(err)) + : "no error"); + if (err) + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Sequential ownership transfer works after release\n"); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + TEST_SAY("SUCCESS: Concurrent thread access correctly rejected\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that record headers are preserved end-to-end through the + * share consumer. + * + * Produce a single record with three headers and verify the share consumer + * receives all of them with the correct values. + */ +static void do_test_headers_preserved(void) { + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + rd_kafka_message_t *msg0; + const char *topic; + const char *group = "share-headers"; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + rd_kafka_headers_t *hdrs = NULL; + rd_kafka_resp_err_t herr; + const void *val; + size_t vsz; + size_t rcvd = 0; + int attempts; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== Headers preserved through share consumer ===\n"); + + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-headers", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Produce a single record with three headers */ + err = rd_kafka_producev( + common_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_KEY("hdr-key", 7), RD_KAFKA_V_VALUE("hdr-value", 9), + RD_KAFKA_V_HEADER("h1", "v1", 2), RD_KAFKA_V_HEADER("h2", "v2", 2), + RD_KAFKA_V_HEADER("h3", "v3", 2), RD_KAFKA_V_END); + TEST_ASSERT(!err, "producev failed: %s", rd_kafka_err2name(err)); + rd_kafka_flush(common_producer, 10 * 1000); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 30; + while (rcvd < 1 && attempts-- > 0) { + rd_kafka_error_t *e; + + rd_kafka_messages_destroy(batch); + batch = NULL; + e = rd_kafka_share_poll(consumer, 2000, &batch); + if (e) + rd_kafka_error_destroy(e); + rcvd = rd_kafka_messages_count(batch); + } + TEST_ASSERT(rcvd == 1, "Expected 1 record, got %zu", rcvd); + + msg0 = rd_kafka_messages_get(batch, 0); + herr = rd_kafka_message_headers(msg0, &hdrs); + TEST_ASSERT(herr == RD_KAFKA_RESP_ERR_NO_ERROR && hdrs, + "Failed to get headers: %s", rd_kafka_err2name(herr)); + + herr = rd_kafka_header_get_last(hdrs, "h1", &val, &vsz); + TEST_ASSERT(herr == RD_KAFKA_RESP_ERR_NO_ERROR, "h1 not present: %s", + rd_kafka_err2name(herr)); + TEST_ASSERT(vsz == 2 && memcmp(val, "v1", 2) == 0, "h1 value mismatch"); + + herr = rd_kafka_header_get_last(hdrs, "h2", &val, &vsz); + TEST_ASSERT(herr == RD_KAFKA_RESP_ERR_NO_ERROR, "h2 not present: %s", + rd_kafka_err2name(herr)); + TEST_ASSERT(vsz == 2 && memcmp(val, "v2", 2) == 0, "h2 value mismatch"); + + herr = rd_kafka_header_get_last(hdrs, "h3", &val, &vsz); + TEST_ASSERT(herr == RD_KAFKA_RESP_ERR_NO_ERROR, "h3 not present: %s", + rd_kafka_err2name(herr)); + TEST_ASSERT(vsz == 2 && memcmp(val, "v3", 2) == 0, "h3 value mismatch"); + + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + TEST_SAY("SUCCESS: Headers preserved through share consumer\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that a share consumer with a small session-level fetch.max.bytes + * receives records one by one per fetch. + * + * - With fetch.max.bytes=1500 and 1000-byte records each in their own + * RecordBatch (custom producer, batch.num.messages=1), every + * ShareFetch response can hold at most one record. + * - All 100 records are received (consumer correctly loops across + * multiple ShareFetch RPCs). + * - Every non-empty consume_batch returns exactly one record (one batch + * per ShareFetch response, due to the cap). + */ +static void do_test_fetch_max_bytes_small(void) { + const char *group = "share-small-fetch"; + const char *topic; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_conf_t *prod_conf; + rd_kafka_t *prod; + rd_kafka_topic_partition_list_t *subs; + char errstr[512]; + const int msgcnt = 10; + const int msgsize = 1000; + int consumed = 0; + int attempts; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "=== Small session-level fetch.max.bytes (1500) with 1000-byte " + "msgs ===\n"); + + topic = test_mk_topic_name("0171-small-fetch", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Custom producer that emits one RecordBatch per message so the + * broker has many small batches to choose from when responding to + * ShareFetch */ + test_conf_init(&prod_conf, NULL, 60); + rd_kafka_conf_set_dr_msg_cb(prod_conf, test_dr_msg_cb); + rd_kafka_conf_set(prod_conf, "linger.ms", "0", errstr, sizeof(errstr)); + rd_kafka_conf_set(prod_conf, "batch.num.messages", "1", errstr, + sizeof(errstr)); + prod = test_create_handle(RD_KAFKA_PRODUCER, prod_conf); + + test_produce_msgs2(prod, topic, 0, 0, 0, msgcnt, NULL, msgsize); + rd_kafka_flush(prod, 30 * 1000); + rd_kafka_destroy(prod); + + /* Create share consumer with a small session-level fetch cap. + * For share consumers `message.max.bytes` does not floor + * `fetch.max.bytes` (and is rejected), so set `fetch.max.bytes` + * directly to 1500. */ + test_conf_init(&conf, NULL, 60); + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "fetch.max.bytes", "1500", errstr, + sizeof(errstr)); + + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create share consumer: %s", errstr); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* All 100 records must arrive one by one per fetch */ + attempts = 30; + while (consumed < msgcnt && attempts-- > 0) { + int batch_cnt = 0; + test_share_poll(consumer, 2000, NULL, 0, &batch_cnt); + consumed += batch_cnt; + if (batch_cnt > 0) { + TEST_ASSERT(batch_cnt == 1, + "Expected 1 record per batch with small " + "fetch.max.bytes, " + "got %d", + batch_cnt); + } + } + + TEST_ASSERT(consumed == msgcnt, + "Expected %d records with small fetch.max.bytes, " + "consumed %d", + msgcnt, consumed); + + TEST_SAY("SUCCESS: small fetch.max.bytes - consumed %d records\n", + consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Verify the share consumer receives records that individually + * exceed fetch.max.bytes, the broker doesnt stall consumption. + * + * What is tested: + * - Produce a mix of 10 KB and 500 KB records on a topic. + * - Configure the consumer with fetch.max.bytes=1500 — well below every + * record's on-wire size. + * - All records must still arrive. The broker is required to return the + * first batch in a non-empty partition even when it exceeds the cap + * to guarantee consumer progress; the client side must accept these + * oversized responses. + */ +static void do_test_record_larger_than_fetch_max_bytes(void) { + const char *group = "share-large-record"; + const char *topic; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + char errstr[512]; + const int MAX_BYTES = 100000; + const int small_cnt = 5; + const int large_cnt = 2; + const int total = small_cnt + large_cnt; + int consumed = 0; + int attempts; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== Records larger than fetch.message.max.bytes (%d) ===\n", + MAX_BYTES); + + topic = test_mk_topic_name("0171-large-record", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Produce small messages then large ones */ + test_produce_msgs2(common_producer, topic, 0, 0, 0, small_cnt, NULL, + MAX_BYTES / 10); + test_produce_msgs2(common_producer, topic, 0, 0, small_cnt, large_cnt, + NULL, MAX_BYTES * 5); + rd_kafka_flush(common_producer, 60 * 1000); + + test_conf_init(&conf, NULL, 60); + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "fetch.max.bytes", "1500", errstr, + sizeof(errstr)); + + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create share consumer: %s", errstr); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 30; + while (consumed < total && attempts-- > 0) { + int batch_cnt = 0; + test_share_poll(consumer, 3000, NULL, 0, &batch_cnt); + consumed += batch_cnt; + if (batch_cnt > 0) + TEST_SAY("Progress: %d/%d\n", consumed, total); + } + + TEST_ASSERT(consumed == total, + "Expected %d records (incl. oversized), consumed %d", total, + consumed); + + TEST_SAY("SUCCESS: oversized record - consumed %d records\n", consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that records held by a consumer that closes mid-flight + * are released back to other consumers in the group. + * + * Create 4 share consumers in the same group, produce a known set of + * records, have consumer 0 fetch a batch without acknowledging and then + * close. The other consumers should eventually receive every record + * (including the ones released by consumer 0). + */ +static void do_test_consumer_close_releases_to_group(void) { + const char *group = "share-close-releases"; + const char *topic; + rd_kafka_share_t *consumers[4]; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + const int total_msgs = 200; + int c0_fetched = 0; + int total_consumed = 0; + int per_consumer[4] = {0}; + int attempts; + int i; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "=== Failing consumer releases unacked records to group ===\n"); + + topic = test_mk_topic_name("0171-close-release", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, total_msgs); + + test_share_set_auto_offset_reset(group, "earliest"); + + for (i = 0; i < 4; i++) + consumers[i] = test_create_share_consumer(group, NULL); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + for (i = 0; i < 4; i++) + rd_kafka_share_subscribe(consumers[i], subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consumer 0 fetches a batch WITHOUT acknowledging, then closes. */ + attempts = 30; + while (c0_fetched == 0 && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumers[0], 2000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + c0_fetched++; + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(c0_fetched > 0, + "Consumer 0 should have fetched some records, got %d", + c0_fetched); + + TEST_SAY("Consumer 0 fetched %d records (unacked), closing now\n", + c0_fetched); + /* Close without acknowledging - records should be released */ + test_share_consumer_close(consumers[0]); + test_share_destroy(consumers[0]); + consumers[0] = NULL; + + /* Drain the rest with consumers 1..3 */ + attempts = 200; + while (total_consumed < total_msgs && attempts-- > 0) { + for (i = 1; i < 4; i++) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumers[i], 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) { + per_consumer[i]++; + total_consumed++; + } + } + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY( + "Distribution: c0=%d (lost on close), c1=%d, c2=%d, c3=%d " + "(total %d)\n", + c0_fetched, per_consumer[1], per_consumer[2], per_consumer[3], + total_consumed); + + TEST_ASSERT(total_consumed == total_msgs, + "Expected exactly %d records on the survivors after " + "consumer 0's records were released, got %d", + total_msgs, total_consumed); + + for (i = 1; i < 4; i++) { + test_share_consumer_close(consumers[i]); + test_share_destroy(consumers[i]); + } + + TEST_SAY("SUCCESS: closed consumer's records redelivered to group\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Partition count increased during consume. + * + * 1 consumer, topic starts with 2 partitions, grown to 5. + * Produce N records to {0,1} -> consume -> grow to 5 -> produce N to + * {2,3,4} -> continue consuming -> expect 5*N total. + */ +static void do_test_partition_increase_during_consume(void) { + const char *group = "share-partition-increase"; + const char *topic; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_conf_t *prod_conf; + rd_kafka_t *producer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + const int msgs_per_partition = 200; + const int initial_partitions = 2; + const int grown_partitions = 5; + const int initial_expected = initial_partitions * msgs_per_partition; + const int total_expected = grown_partitions * msgs_per_partition; + int consumed = 0; + int attempts; + int p; + rd_kafka_resp_err_t cp_err; + char errstr[512]; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== Partition count increase during consume (%d -> %d) ===\n", + initial_partitions, grown_partitions); + + topic = test_mk_topic_name("0171-partition-increase", 1); + test_create_topic_wait_exists(NULL, topic, initial_partitions, -1, + 60 * 1000); + + /* Custom producer with a short metadata refresh interval so its + * cache picks up the post-CreatePartitions partition count on its + * own — avoids the explicit test_wait_metadata_update probe. */ + test_conf_init(&prod_conf, NULL, 60); + rd_kafka_conf_set_dr_msg_cb(prod_conf, test_dr_msg_cb); + rd_kafka_conf_set(prod_conf, "topic.metadata.refresh.interval.ms", + "500", errstr, sizeof(errstr)); + producer = test_create_handle(RD_KAFKA_PRODUCER, prod_conf); + + /* Produce to initial partitions only */ + for (p = 0; p < initial_partitions; p++) + test_produce_msgs_simple(producer, topic, p, + msgs_per_partition); + + /* Lower the consumer's metadata refresh interval (default 5 min) so + * it picks up the new partition count quickly without a long + * hardcoded sleep below. */ + test_conf_init(&conf, NULL, 60); + rd_kafka_conf_set(conf, "group.id", group, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "topic.metadata.refresh.interval.ms", "500", + errstr, sizeof(errstr)); + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create share consumer: %s", errstr); + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume from the initial partitions */ + attempts = 100; + while (consumed < initial_expected && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + } + TEST_SAY("Pre-grow consumed: %d/%d\n", consumed, initial_expected); + TEST_ASSERT(consumed == initial_expected, + "Expected >= %d msgs from initial partitions, got %d", + initial_expected, consumed); + + /* Grow topic partitions from 2 -> 5 */ + TEST_SAY("Growing partitions to %d via CreatePartitions\n", + grown_partitions); + cp_err = test_CreatePartitions_simple(common_admin, NULL, topic, + grown_partitions, NULL); + TEST_ASSERT(!cp_err, "CreatePartitions failed: %s", + rd_kafka_err2str(cp_err)); + + /* Both producer and consumer have topic.metadata.refresh.interval + * .ms=500, so their local metadata caches pick up the new partition + * count within ~500 ms via the background refresh. One second is + * plenty for both: the producer can route to the new partitions, + * and the consumer can be assigned them on the next + * ShareGroupHeartbeat cycle. */ + rd_sleep(1); + + /* Produce to the newly added partitions */ + for (p = initial_partitions; p < grown_partitions; p++) + test_produce_msgs_simple(producer, topic, p, + msgs_per_partition); + + /* Continue consuming, expecting records from new partitions */ + attempts = 200; + while (consumed < total_expected && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + consumed++; + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == total_expected, + "Expected exactly %d total records after partition grow, " + "got %d", + total_expected, consumed); + + TEST_SAY("SUCCESS: partition count grow %d->%d, consumed %d records\n", + initial_partitions, grown_partitions, consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Zero-byte payload and zero-byte key. + * + * Verifies that the share consumer correctly preserves zero-length + * key/value buffers (len == 0, pointer non-NULL) and NULL key/value. + */ +static void do_test_zero_byte_payload_and_key(void) { + const char *group = "share-zero-byte"; + const char *topic; + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_resp_err_t err; + size_t rcvd = 0; + int attempts; + int saw_empty_kv = 0; + int saw_key_only = 0; + int saw_value_only = 0; + int saw_null_kv = 0; + size_t i; + rd_bool_t key_null; + rd_bool_t key_empty; + rd_bool_t val_null; + rd_bool_t val_empty; + rd_bool_t key_is_empty_or_null; + rd_bool_t val_is_empty_or_null; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== Zero-byte payload and key ===\n"); + + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-zero-byte", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Empty key, empty value (zero-length, non-NULL pointers) */ + err = rd_kafka_producev(common_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_KEY("", 0), RD_KAFKA_V_VALUE("", 0), + RD_KAFKA_V_END); + TEST_ASSERT(!err, "produce empty kv failed: %s", + rd_kafka_err2name(err)); + + /* Non-empty key, empty value */ + err = rd_kafka_producev(common_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_KEY("k", 1), RD_KAFKA_V_VALUE("", 0), + RD_KAFKA_V_END); + TEST_ASSERT(!err, "produce key-only failed: %s", + rd_kafka_err2name(err)); + + /* Empty key, non-empty value */ + err = rd_kafka_producev(common_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_KEY("", 0), RD_KAFKA_V_VALUE("v", 1), + RD_KAFKA_V_END); + TEST_ASSERT(!err, "produce value-only failed: %s", + rd_kafka_err2name(err)); + + /* NULL key, NULL value */ + err = rd_kafka_producev(common_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_KEY(NULL, 0), + RD_KAFKA_V_VALUE(NULL, 0), RD_KAFKA_V_END); + TEST_ASSERT(!err, "produce null kv failed: %s", rd_kafka_err2name(err)); + + rd_kafka_flush(common_producer, 10 * 1000); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 30; + while (rcvd < 4 && attempts-- > 0) { + size_t batch_rcvd = 0; + rd_kafka_error_t *e; + + rd_kafka_messages_destroy(batch); + batch = NULL; + e = rd_kafka_share_poll(consumer, 2000, &batch); + if (e) + rd_kafka_error_destroy(e); + batch_rcvd = rd_kafka_messages_count(batch); + + for (i = 0; i < batch_rcvd; i++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, i); + + key_null = msg->key == NULL; + key_empty = msg->key_len == 0 && !key_null; + val_null = msg->payload == NULL; + val_empty = msg->len == 0 && !val_null; + + /* Brokers may normalize empty buffers to NULL; accept + * either. */ + key_is_empty_or_null = key_null || key_empty; + val_is_empty_or_null = val_null || val_empty; + + if (key_is_empty_or_null && val_is_empty_or_null) { + /* Could be "empty kv" or "null kv" record - + * count both as the "no content" class. */ + if (key_null && val_null) + saw_null_kv++; + else + saw_empty_kv++; + } else if (!key_is_empty_or_null && + val_is_empty_or_null) { + TEST_ASSERT(msg->key_len == 1 && + memcmp(msg->key, "k", 1) == 0, + "key-only record: key mismatch"); + saw_key_only++; + } else if (key_is_empty_or_null && + !val_is_empty_or_null) { + TEST_ASSERT( + msg->len == 1 && + memcmp(msg->payload, "v", 1) == 0, + "value-only record: value mismatch"); + saw_value_only++; + } + } + rcvd += batch_rcvd; + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(rcvd == 4, "Expected 4 records, got %zu", rcvd); + + TEST_SAY("Counts: empty_kv=%d null_kv=%d key_only=%d value_only=%d\n", + saw_empty_kv, saw_null_kv, saw_key_only, saw_value_only); + + TEST_ASSERT(saw_empty_kv + saw_null_kv == 2, + "Expected 2 'no content' records, got %d", + saw_empty_kv + saw_null_kv); + TEST_ASSERT(saw_key_only == 1, "Expected 1 key-only record, got %d", + saw_key_only); + TEST_ASSERT(saw_value_only == 1, "Expected 1 value-only record, got %d", + saw_value_only); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + TEST_SAY("SUCCESS: zero-byte payload/key preserved\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Consume + ack a round of messages produced with the given + * compression codec. + */ +static void do_test_compression_codec(const char *codec) { + const char *group_prefix = "share-codec-"; + char group[128]; + const char *topic; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_t *producer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + const int msg_cnt = 200; + const int msg_sz = 256; + int consumed = 0; + int attempts; + + TEST_SAY("\n"); + TEST_SAY("=== Compression codec: %s ===\n", codec); + + rd_snprintf(group, sizeof(group), "%s%s", group_prefix, codec); + topic = test_mk_topic_name(codec, 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Producer configured with the requested compression codec. + * test_produce_msgs2 → test_wait_delivery uses a counter that's + * decremented inside test_dr_msg_cb, so the dr callback must + * be wired up explicitly when we create our own producer (only + * test_create_producer() sets it for us). */ + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "compression.type", codec); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + producer = test_create_handle(RD_KAFKA_PRODUCER, conf); + + test_produce_msgs2(producer, topic, 0, 0, 0, msg_cnt, NULL, msg_sz); + rd_kafka_flush(producer, 30 * 1000); + rd_kafka_destroy(producer); + + consumer = test_create_share_consumer(group, NULL); + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 100; + while (consumed < msg_cnt && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(consumer, 2000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) { + TEST_ASSERT(msg->len == (size_t)msg_sz, + "[%s] msg size %zu != expected %d", + codec, msg->len, msg_sz); + consumed++; + } + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(consumed == msg_cnt, + "[%s] expected %d records, consumed %d", codec, msg_cnt, + consumed); + + TEST_SAY("SUCCESS: codec %s - consumed %d records\n", codec, consumed); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); +} + + +/** + * @brief Run consume+ack roundtrip for every supported producer + * compression codec. + */ +static void do_test_all_compression_codecs(void) { + SUB_TEST(); + do_test_compression_codec("none"); + do_test_compression_codec("gzip"); + do_test_compression_codec("snappy"); + do_test_compression_codec("lz4"); + do_test_compression_codec("zstd"); + SUB_TEST_PASS(); +} + + +/** + * @brief Records with 100 headers and one very large header value. + * + * Verifies the share consumer preserves a large header count and large + * per-header value sizes end-to-end. + */ +static void do_test_many_and_large_headers(void) { + const char *group = "share-many-headers"; + const char *topic; + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_resp_err_t err; + const int hdr_cnt = 100; + const size_t big_hdr_size = 64 * 1024; + char *big_hdr_value; + size_t rcvd = 0; + int attempts; + size_t i; + rd_bool_t verified_many = rd_false; + rd_bool_t verified_big = rd_false; + rd_kafka_headers_t *hdrs; + char name[32]; + char value[32]; + rd_kafka_headers_t *msg_hdrs = NULL; + rd_kafka_resp_err_t herr; + size_t n; + size_t j; + char expected[32]; + const void *val; + size_t vsz; + size_t k; + const char *bytes; + char expected_byte; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "=== Many headers (%d) and one large header value (%zu B) ===\n", + hdr_cnt, big_hdr_size); + + consumer = test_create_share_consumer(group, NULL); + topic = test_mk_topic_name("0171-many-headers", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Build the producev arg list for a record with `hdr_cnt` headers. + * Each header name is "h-" and each value is "v-". */ + hdrs = rd_kafka_headers_new(hdr_cnt); + for (i = 0; i < (size_t)hdr_cnt; i++) { + rd_snprintf(name, sizeof(name), "h-%zu", i); + rd_snprintf(value, sizeof(value), "v-%zu", i); + rd_kafka_header_add(hdrs, name, -1, value, -1); + } + + err = rd_kafka_producev(common_producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_KEY("many", 4), + RD_KAFKA_V_VALUE("many-payload", 12), + RD_KAFKA_V_HEADERS(hdrs), RD_KAFKA_V_END); + TEST_ASSERT(!err, "producev many headers failed: %s", + rd_kafka_err2name(err)); + + /* Record with one very large header value. */ + big_hdr_value = rd_malloc(big_hdr_size); + for (i = 0; i < big_hdr_size; i++) + big_hdr_value[i] = (char)('A' + (i % 26)); + + err = rd_kafka_producev( + common_producer, RD_KAFKA_V_TOPIC(topic), RD_KAFKA_V_KEY("big", 3), + RD_KAFKA_V_VALUE("big-payload", 11), + RD_KAFKA_V_HEADER("big-hdr", big_hdr_value, big_hdr_size), + RD_KAFKA_V_END); + TEST_ASSERT(!err, "producev big header failed: %s", + rd_kafka_err2name(err)); + rd_free(big_hdr_value); + + rd_kafka_flush(common_producer, 30 * 1000); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 30; + while (rcvd < 2 && attempts-- > 0) { + size_t batch_rcvd = 0; + rd_kafka_error_t *e; + + rd_kafka_messages_destroy(batch); + batch = NULL; + e = rd_kafka_share_poll(consumer, 2000, &batch); + if (e) + rd_kafka_error_destroy(e); + batch_rcvd = rd_kafka_messages_count(batch); + + for (i = 0; i < batch_rcvd; i++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, i); + msg_hdrs = NULL; + herr = rd_kafka_message_headers(msg, &msg_hdrs); + TEST_ASSERT(herr == RD_KAFKA_RESP_ERR_NO_ERROR && + msg_hdrs, + "headers not available: %s", + rd_kafka_err2name(herr)); + + if (msg->key_len == 4 && + memcmp(msg->key, "many", 4) == 0) { + n = rd_kafka_header_cnt(msg_hdrs); + + TEST_ASSERT(n == (size_t)hdr_cnt, + "many-headers record: expected %d " + "headers, got %zu", + hdr_cnt, n); + + /* Verify every header name+value present. */ + for (j = 0; j < (size_t)hdr_cnt; j++) { + rd_snprintf(name, sizeof(name), "h-%zu", + j); + rd_snprintf(expected, sizeof(expected), + "v-%zu", j); + herr = rd_kafka_header_get_last( + msg_hdrs, name, &val, &vsz); + TEST_ASSERT( + herr == RD_KAFKA_RESP_ERR_NO_ERROR, + "%s missing: %s", name, + rd_kafka_err2name(herr)); + TEST_ASSERT( + vsz == strlen(expected) && + memcmp(val, expected, vsz) == 0, + "%s value mismatch", name); + } + verified_many = rd_true; + } else if (msg->key_len == 3 && + memcmp(msg->key, "big", 3) == 0) { + herr = rd_kafka_header_get_last( + msg_hdrs, "big-hdr", &val, &vsz); + TEST_ASSERT(herr == RD_KAFKA_RESP_ERR_NO_ERROR, + "big-hdr missing: %s", + rd_kafka_err2name(herr)); + TEST_ASSERT(vsz == big_hdr_size, + "big-hdr size mismatch: expected " + "%zu, got %zu", + big_hdr_size, vsz); + + /* Spot-check the content pattern at a few + * offsets. */ + bytes = (const char *)val; + for (k = 0; k < vsz; k += vsz / 16) { + expected_byte = (char)('A' + (k % 26)); + TEST_ASSERT( + bytes[k] == expected_byte, + "big-hdr byte %zu: expected %c, " + "got %c", + k, expected_byte, bytes[k]); + } + verified_big = rd_true; + } + } + rcvd += batch_rcvd; + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(rcvd == 2, "Expected 2 records, got %zu", rcvd); + + TEST_ASSERT(verified_many, "many-headers record was not verified"); + TEST_ASSERT(verified_big, "big-header record was not verified"); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + TEST_SAY("SUCCESS: many headers + large header value preserved\n"); + + SUB_TEST_PASS(); +} + + +int main_0171_share_consumer_consume(int argc, char **argv) { + /* Create common handles for all tests */ + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + test_timeout_set(600); + + /* Single-consumer tests */ + do_test_single_consumer_single_topic_single_partition(); /* Single + consumer, + single topic, + single + partition */ + do_test_single_consumer_single_topic_multiple_partitions(); /* Single + consumer, + single + topic, multi + partitions + */ + do_test_single_consumer_multiple_topic_single_partition(); /* Single + consumer, + multi topic, + single + partition + each */ + do_test_single_consumer_multiple_topic_multiple_partitions(); /* Single + consumer, + multi + topics, + multi + partitions + each */ + + /* Multi-consumer tests */ + do_test_multiple_consumers_single_topic_single_partition(); /* Multi + consumer + sharing + single + partition */ + do_test_multiple_consumers_single_topic_multiple_partitions(); /* Multi + consumer, + multi + partition + */ + do_test_multiple_consumers_multiple_topics_multiple_partitions(); /* Full + matrix: + multi + everything + */ + + /* High volume tests */ + do_test_high_volume_10k_messages(); + do_test_high_volume_50k_multi_partition(); + + /* Multi-topic tests (triggers multiple fetch responses) */ + do_test_many_topics_15(); + do_test_many_topics_10_multi_partition(); + + /* Rapid produce/consume tests */ + do_test_rapid_produce_consume_cycles(); + + /* Edge case tests */ + do_test_empty_then_produce(); + do_test_sparse_partitions(); + + /* Callback tests */ + do_test_poll_callback_piggybacked_acks(); + + /* Concurrency tests */ + do_test_concurrent_thread_access_rejected(); + + /* Headers and fetch byte-limit tests */ + do_test_headers_preserved(); + do_test_fetch_max_bytes_small(); + do_test_record_larger_than_fetch_max_bytes(); + + /* Failing consumer releases to group */ + do_test_consumer_close_releases_to_group(); + + /* Topology change: grow partitions while consuming */ + do_test_partition_increase_during_consume(); + + /* Payload edge cases */ + do_test_zero_byte_payload_and_key(); + do_test_all_compression_codecs(); + do_test_many_and_large_headers(); + + + /* Cleanup common handles */ + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0172-share_consumer_acknowledge.c b/lib/librdkafka-2.15.0/tests/0172-share_consumer_acknowledge.c new file mode 100644 index 00000000000..69629703826 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0172-share_consumer_acknowledge.c @@ -0,0 +1,1919 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +/** Common producer reused across all tests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all tests. */ +static rd_kafka_t *common_admin; + +/** + * @brief Share consumer acknowledge API integration tests. + * + * Tests the acknowledge APIs (ACCEPT, REJECT, RELEASE) with real/mock broker. + * + * Expected Behavior: + * - RELEASE: Records redelivered to the same or another consumer in the same + * group. + * - REJECT: NOT redelivered + * - ACCEPT: Records committed, NOT redelivered + * + * All tests use share.acknowledgement.mode = "explicit" + */ + +#define MAX_TOPICS 16 +#define MAX_PARTITIONS 8 +#define MAX_CONSUMERS 4 +#define MAX_MSGS_PER_PART 100 +#define BATCH_SIZE 10000 + +/** + * @brief Test configuration structure + */ +typedef struct { + const char *test_name; + int topic_cnt; + int partitions[MAX_TOPICS]; + int msgs_per_partition; + int consumer_cnt; + int poll_timeout_ms; + int max_attempts; + int total_msgs; + rd_bool_t use_random_acks; /**< Generate random acks at runtime */ + rd_kafka_share_AcknowledgeType_t + single_ack_type; /**< Ack type when not using random */ +} ack_test_config_t; + +/** + * @brief Test state structure + */ +typedef struct { + rd_kafka_share_t *consumers[MAX_CONSUMERS]; + char *topic_names[MAX_TOPICS]; + int64_t original_offsets[1000]; + int original_cnt; + rd_kafka_topic_partition_list_t *released_msgs; /**< RELEASE'd msgs */ + rd_kafka_topic_partition_list_t + *redelivered_msgs; /**< Redelivered msgs */ + int msgs_produced; + int msgs_consumed; + int msgs_redelivered; + char group_name[300]; + /* Random mode counters */ + int msgs_accepted; + int msgs_rejected; + int msgs_released; +} ack_test_state_t; + + +/** + * @brief Generate random ack type with roughly equal distribution + */ +static rd_kafka_share_AcknowledgeType_t get_random_ack_type(void) { + return (rd_kafka_share_AcknowledgeType_t)jitter(1, 3); +} + +/** + * @brief Create topics and produce messages + */ +static void setup_topics_and_produce(ack_test_config_t *config, + ack_test_state_t *state) { + int t, p; + int msgs_per_partition; + int total_partitions = 0; + + state->msgs_produced = 0; + + /* Calculate msgs_per_partition based on mode */ + if (config->use_random_acks && config->total_msgs > 0) { + for (t = 0; t < config->topic_cnt; t++) + total_partitions += config->partitions[t]; + msgs_per_partition = config->total_msgs / total_partitions; + } else { + msgs_per_partition = config->msgs_per_partition; + } + + for (t = 0; t < config->topic_cnt; t++) { + state->topic_names[t] = + rd_strdup(test_mk_topic_name("0172-ack-test", 1)); + + test_create_topic_wait_exists(NULL, state->topic_names[t], + config->partitions[t], -1, + 60 * 1000); + + for (p = 0; p < config->partitions[t]; p++) { + test_produce_msgs_simple(common_producer, + state->topic_names[t], p, + msgs_per_partition); + state->msgs_produced += msgs_per_partition; + } + + TEST_SAY("Topic '%s': %d partition(s), %d msgs/partition\n", + state->topic_names[t], config->partitions[t], + msgs_per_partition); + } + + TEST_SAY("Produced %d messages total\n", state->msgs_produced); +} + +/** + * @brief Subscribe consumers to topics + */ +static void subscribe_consumers(ack_test_config_t *config, + ack_test_state_t *state) { + rd_kafka_topic_partition_list_t *subs; + int t, i; + + test_share_set_auto_offset_reset(state->group_name, "earliest"); + + subs = rd_kafka_topic_partition_list_new(config->topic_cnt); + for (t = 0; t < config->topic_cnt; t++) { + rd_kafka_topic_partition_list_add(subs, state->topic_names[t], + RD_KAFKA_PARTITION_UA); + } + + for (i = 0; i < config->consumer_cnt; i++) { + rd_kafka_share_subscribe(state->consumers[i], subs); + } + + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("Subscribed %d consumer(s) to %d topic(s)\n", + config->consumer_cnt, config->topic_cnt); +} + +/** + * @brief Find message index in list by topic+partition+offset + * @returns Index if found, -1 otherwise + */ +static int find_message_in_list(rd_kafka_topic_partition_list_t *list, + const char *topic, + int32_t partition, + int64_t offset) { + int i; + for (i = 0; i < list->cnt; i++) { + rd_kafka_topic_partition_t *elem = &list->elems[i]; + if (strcmp(elem->topic, topic) == 0 && + elem->partition == partition && elem->offset == offset) + return i; + } + return -1; +} + +/** + * @brief Remove message from list by index (swap with last element) + */ +static void remove_message_from_list_at(rd_kafka_topic_partition_list_t *list, + int idx) { + if (idx < 0 || idx >= list->cnt) + return; + + /* Free the topic string being removed */ + rd_free(list->elems[idx].topic); + + /* Swap with last element and decrement count */ + if (idx < list->cnt - 1) { + list->elems[idx] = list->elems[list->cnt - 1]; + /* Null out the moved element to prevent double-free */ + list->elems[list->cnt - 1].topic = NULL; + } + list->cnt--; +} + +/** + * @brief Handle a redelivered message (delivery_count == 2). + * + * Verifies the message was previously RELEASE'd (in released_msgs), + * removes it from released_msgs, and acknowledges as ACCEPT. + */ +static void handle_redelivered_message(ack_test_state_t *state, + rd_kafka_message_t *msg) { + int released_idx; + + released_idx = find_message_in_list(state->released_msgs, + rd_kafka_topic_name(msg->rkt), + msg->partition, msg->offset); + + TEST_ASSERT(released_idx >= 0, + "Redelivered message (delivery_count=2) not found in " + "released_msgs: topic=%s, partition=%d, offset=%" PRId64, + rd_kafka_topic_name(msg->rkt), msg->partition, msg->offset); + + remove_message_from_list_at(state->released_msgs, released_idx); + state->msgs_redelivered++; + + rd_kafka_share_acknowledge_type(state->consumers[0], msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); +} + +/** + * @brief Determine ack type based on config. + */ +static rd_kafka_share_AcknowledgeType_t +determine_ack_type(ack_test_config_t *config) { + if (config->use_random_acks) + return get_random_ack_type(); + return config->single_ack_type; +} + +/** + * @brief Track ack type in state and add to released list if RELEASE. + */ +static void track_ack_type(ack_test_state_t *state, + rd_kafka_message_t *msg, + rd_kafka_share_AcknowledgeType_t ack_type) { + if (ack_type == RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE) { + rd_kafka_topic_partition_t *rktpar; + rktpar = rd_kafka_topic_partition_list_add( + state->released_msgs, rd_kafka_topic_name(msg->rkt), + msg->partition); + rktpar->offset = msg->offset; + state->msgs_released++; + } else if (ack_type == RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT) { + state->msgs_accepted++; + } else { + state->msgs_rejected++; + } +} + +/** + * @brief Consume messages and apply acknowledgements based on config + */ +static void consume_and_acknowledge(ack_test_config_t *config, + ack_test_state_t *state) { + rd_kafka_messages_t *batch = NULL; + int poll_timeout = + config->poll_timeout_ms > 0 ? config->poll_timeout_ms : 3000; + int attempts = config->max_attempts > 0 ? config->max_attempts : 50; + size_t total_consumed = 0; + + if (config->use_random_acks) { + poll_timeout = 5000; + attempts = 200 + (config->total_msgs / 1000) * 50; + } + + state->original_cnt = 0; + state->msgs_consumed = 0; + state->msgs_accepted = 0; + state->msgs_rejected = 0; + state->msgs_released = 0; + state->msgs_redelivered = 0; + + TEST_SAY("Consuming %d messages%s...\n", state->msgs_produced, + config->use_random_acks ? " with random acks" : ""); + + while ((int)total_consumed < state->msgs_produced && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + err = rd_kafka_share_poll(state->consumers[0], poll_timeout, + &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + rd_kafka_share_AcknowledgeType_t ack_type; + rd_kafka_resp_err_t ack_err; + int16_t delivery_count = + rd_kafka_message_delivery_count(msg); + + /* Error messages must use acknowledge_offset API */ + if (msg->err) { + ack_type = determine_ack_type(config); + ack_err = rd_kafka_share_acknowledge_offset( + state->consumers[0], + rd_kafka_topic_name(msg->rkt), + msg->partition, msg->offset, ack_type); + TEST_ASSERT( + ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "acknowledge_offset failed for error msg: " + "%s", + rd_kafka_err2str(ack_err)); + continue; + } + + /* Redelivered message (delivery_count == 2) */ + if (delivery_count == 2) { + handle_redelivered_message(state, msg); + continue; + } + + /* First delivery */ + TEST_ASSERT(delivery_count == 1, + "Expected delivery_count=1, got %d", + delivery_count); + + ack_type = determine_ack_type(config); + track_ack_type(state, msg, ack_type); + + ack_err = rd_kafka_share_acknowledge_type( + state->consumers[0], msg, ack_type); + TEST_ASSERT( + ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Acknowledge failed: %s (topic=%s, partition=%d, " + "offset=%" PRId64 ", type=%d)", + rd_kafka_err2str(ack_err), + rd_kafka_topic_name(msg->rkt), msg->partition, + msg->offset, ack_type); + + if (state->original_cnt < 1000) + state->original_offsets[state->original_cnt++] = + msg->offset; + + total_consumed++; + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + if (config->use_random_acks) { + if (total_consumed % 500 == 0 || rcvd > 0) + TEST_SAY( + "Progress: %zu/%d (A:%d R:%d L:%d " + "redeliv:%d)\n", + total_consumed, state->msgs_produced, + state->msgs_accepted, state->msgs_rejected, + state->msgs_released, + state->msgs_redelivered); + } else { + TEST_SAY("Progress: %zu/%d\n", total_consumed, + state->msgs_produced); + } + } + + state->msgs_consumed = (int)total_consumed; + TEST_ASSERT(state->msgs_consumed == state->msgs_produced, + "Expected to consume %d messages, got %d", + state->msgs_produced, state->msgs_consumed); + + if (config->use_random_acks) + TEST_SAY( + "Consumed %d: ACCEPT=%d, REJECT=%d, RELEASE=%d, " + "redelivered=%d\n", + state->msgs_consumed, state->msgs_accepted, + state->msgs_rejected, state->msgs_released, + state->msgs_redelivered); +} + +/** + * @brief Poll for redelivered messages after acknowledgements + */ +static void poll_for_redelivery(ack_test_config_t *config, + ack_test_state_t *state) { + rd_kafka_messages_t *batch = NULL; + int poll_timeout = + config->poll_timeout_ms > 0 ? config->poll_timeout_ms : 3000; + int attempts = 10; + int expected_count = state->msgs_released; + + /* Use higher timeout/attempts for random mode */ + if (config->use_random_acks) { + poll_timeout = 5000; + /* Scale attempts based on expected redeliveries */ + attempts = 100 + (expected_count / 500) * 50; + } + + /* + * In random mode, some redeliveries may have already been handled + * during consume_and_acknowledge. Don't reset the counter. + * For non-random mode, reset as before. + */ + if (!config->use_random_acks) + state->msgs_redelivered = 0; + + TEST_SAY( + "Polling for redelivered messages (expecting %d, have %d)...\n", + expected_count, state->msgs_redelivered); + + while (attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + err = rd_kafka_share_poll(state->consumers[0], poll_timeout, + &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) { + rd_kafka_topic_partition_t *rktpar; + int16_t delivery_count = + rd_kafka_message_delivery_count(msg); + const char *msg_topic = + rd_kafka_topic_name(msg->rkt); + int32_t msg_partition = msg->partition; + int64_t msg_offset = msg->offset; + int released_idx; + + /* Verify delivery_count >= 2 on redelivery */ + TEST_ASSERT( + delivery_count >= 2, + "Expected delivery_count>=2 on redelivery, " + "got %d", + delivery_count); + + /* Verify message was in released list */ + released_idx = find_message_in_list( + state->released_msgs, msg_topic, + msg_partition, msg_offset); + + TEST_ASSERT(released_idx >= 0, + "Redelivered message (topic=%s, " + "partition=%d, offset=%" PRId64 + ") was NOT in RELEASE'd list", + msg_topic, msg_partition, + msg_offset); + + /* Remove from released list */ + remove_message_from_list_at( + state->released_msgs, released_idx); + + /* Track redelivered message */ + rktpar = rd_kafka_topic_partition_list_add( + state->redelivered_msgs, msg_topic, + msg_partition); + rktpar->offset = msg_offset; + + state->msgs_redelivered++; + + rd_kafka_share_acknowledge_type( + state->consumers[0], msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + if (rcvd > 0) { + TEST_SAY("Redelivered so far: %d/%d\n", + state->msgs_redelivered, expected_count); + } + + if (state->msgs_redelivered >= expected_count && + expected_count > 0) { + break; + } + } +} + +/** + * @brief Verify redelivery results + * + * Verification approach: + * - released_msgs is emptied as messages are redelivered + * - Verify redelivered count matches expected count + * - Verify released_msgs is empty (all RELEASE'd messages were redelivered) + */ +static void verify_results(ack_test_config_t *config, ack_test_state_t *state) { + int expected_count = state->msgs_released; + + TEST_SAY("Verifying: consumed=%d, redelivered=%d (expected=%d)\n", + state->msgs_consumed, state->msgs_redelivered, expected_count); + + TEST_ASSERT(state->msgs_redelivered == expected_count, + "Expected %d redelivered messages, got %d", expected_count, + state->msgs_redelivered); + + /* + * All RELEASE'd messages should have been redelivered and removed + * from released_msgs. Verify the list is empty. + */ + TEST_ASSERT(state->released_msgs->cnt == 0, + "Expected all RELEASE'd messages to be redelivered, " + "but %d remain in released list", + state->released_msgs->cnt); + + TEST_SAY("All %d redelivered messages verified correctly\n", + state->msgs_redelivered); +} + +/** + * @brief Cleanup test state + */ +static void cleanup_test(ack_test_config_t *config, ack_test_state_t *state) { + int t, i; + + for (t = 0; t < config->topic_cnt; t++) { + if (state->topic_names[t]) { + rd_free(state->topic_names[t]); + state->topic_names[t] = NULL; + } + } + + for (i = 0; i < config->consumer_cnt; i++) { + if (state->consumers[i]) { + test_share_consumer_close(state->consumers[i]); + test_share_destroy(state->consumers[i]); + state->consumers[i] = NULL; + } + } + + /* Destroy tracking lists */ + if (state->released_msgs) + rd_kafka_topic_partition_list_destroy(state->released_msgs); + if (state->redelivered_msgs) + rd_kafka_topic_partition_list_destroy(state->redelivered_msgs); + + TEST_SAY("Cleanup complete\n"); +} + +/** + * @brief Run a test scenario based on configuration + */ +static int run_ack_test(ack_test_config_t *config) { + ack_test_state_t state = {0}; + int i; + int list_capacity; + char unique_suffix[64]; + char unique_test_name[256]; + + /* Append a per-invocation unique suffix to the test name; the + * group name is derived from it below, so this guarantees a + * fresh share-group on the broker for each run. */ + rd_snprintf(unique_suffix, sizeof(unique_suffix), "rnd%" PRIx64, + test_id_generate()); + rd_snprintf(unique_test_name, sizeof(unique_test_name), "%s-%s", + config->test_name, unique_suffix); + config->test_name = unique_test_name; + + TEST_SAY("\n"); + TEST_SAY( + "============================================================" + "\n"); + TEST_SAY("=== %s ===\n", config->test_name); + TEST_SAY( + "============================================================" + "\n"); + + rd_snprintf(state.group_name, sizeof(state.group_name), "share-%s", + config->test_name); + + /* Initialize tracking lists with larger capacity for random mode */ + list_capacity = config->use_random_acks ? 6000 : 100; + state.released_msgs = rd_kafka_topic_partition_list_new(list_capacity); + state.redelivered_msgs = + rd_kafka_topic_partition_list_new(list_capacity); + + for (i = 0; i < config->consumer_cnt; i++) { + state.consumers[i] = + test_create_share_consumer(state.group_name, "explicit"); + } + + setup_topics_and_produce(config, &state); + subscribe_consumers(config, &state); + consume_and_acknowledge(config, &state); + poll_for_redelivery(config, &state); + verify_results(config, &state); + cleanup_test(config, &state); + + TEST_SAY("=== %s: PASSED ===\n", config->test_name); + return 0; +} + + +/*************************************************************************** + * Core Tests + ***************************************************************************/ + +/** + * @brief RELEASE causes redelivery + */ +static void do_test_release_redelivery(void) { + ack_test_config_t config = { + .test_name = "release-redelivery", + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 5, + .consumer_cnt = 1, + .single_ack_type = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief REJECT prevents redelivery + */ +static void do_test_reject_no_redelivery(void) { + ack_test_config_t config = {.test_name = "reject-no-redelivery", + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 5, + .consumer_cnt = 1, + .single_ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief ACCEPT prevents redelivery + */ +static void do_test_accept_no_redelivery(void) { + ack_test_config_t config = {.test_name = "accept-no-redelivery", + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 5, + .consumer_cnt = 1, + .single_ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + + +/*************************************************************************** + * Error Handling Tests (Standalone) + ***************************************************************************/ + +/** + * @brief Acknowledge with NULL message + */ +static void do_test_ack_null_message(void) { + rd_kafka_share_t *rkshare; + rd_kafka_resp_err_t err; + const char *group = "share-null-msg-test"; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_null_message ===\n"); + + rkshare = test_create_share_consumer(group, "explicit"); + + err = rd_kafka_share_acknowledge(rkshare, NULL); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "Expected INVALID_ARG, got %s", rd_kafka_err2str(err)); + + err = rd_kafka_share_acknowledge_type( + rkshare, NULL, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "Expected INVALID_ARG, got %s", rd_kafka_err2str(err)); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_ack_null_message: PASSED ===\n"); + + SUB_TEST_PASS(); +} + +/** + * @brief Acknowledge with NULL rkshare + */ +static void do_test_ack_null_rkshare(void) { + rd_kafka_resp_err_t err; + rd_kafka_message_t fake_msg = {0}; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_null_rkshare ===\n"); + + /* NULL rkshare is rejected up-front with STATE */ + err = rd_kafka_share_acknowledge(NULL, &fake_msg); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, "Expected STATE, got %s", + rd_kafka_err2str(err)); + + err = rd_kafka_share_acknowledge_type( + NULL, &fake_msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, "Expected STATE, got %s", + rd_kafka_err2str(err)); + + /* Test NULL rkshare directly via acknowledge_offset */ + err = rd_kafka_share_acknowledge_offset( + NULL, "topic", 0, 0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, "Expected STATE, got %s", + rd_kafka_err2str(err)); + + TEST_SAY("=== test_ack_null_rkshare: PASSED ===\n"); + + SUB_TEST_PASS(); +} + +/** + * @brief Acknowledge with invalid type + */ +static void do_test_ack_invalid_type(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + rd_kafka_resp_err_t ack_err; + rd_kafka_topic_partition_list_t *subs; + const char *group = "share-invalid-type-test"; + const char *topic; + size_t rcvd = 0; + int attempts; + rd_kafka_message_t *msg0; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_invalid_type ===\n"); + + rkshare = test_create_share_consumer(group, "explicit"); + topic = test_mk_topic_name("0172-invalid-type", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 1); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 20; + while (rcvd == 0 && attempts-- > 0) { + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + } + + TEST_ASSERT(rcvd == 1, "Expected exactly 1 message, got %zu", rcvd); + + msg0 = rd_kafka_messages_get(batch, 0); + + /* Try invalid type (99) */ + ack_err = rd_kafka_share_acknowledge_type( + rkshare, msg0, (rd_kafka_share_AcknowledgeType_t)99); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "Expected INVALID_ARG for type 99, got %s", + rd_kafka_err2str(ack_err)); + + /* Try GAP type (0) - internal only */ + ack_err = rd_kafka_share_acknowledge_type( + rkshare, msg0, (rd_kafka_share_AcknowledgeType_t)0); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR__INVALID_ARG, + "Expected INVALID_ARG for type 0 (GAP), got %s", + rd_kafka_err2str(ack_err)); + + /* Clean up with valid acknowledge */ + rd_kafka_share_acknowledge_type(rkshare, msg0, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_ack_invalid_type: PASSED ===\n"); + + SUB_TEST_PASS(); +} + +/** + * @brief RELEASE then REJECT - final type is REJECT (no redelivery) + */ +static void do_test_release_then_reject_no_redelivery(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batches[30] = {NULL}; + int batch_cnt = 0; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + rd_kafka_resp_err_t ack_err; + rd_kafka_topic_partition_list_t *subs; + const char *group = "share-release-then-reject"; + const char *topic; + size_t rcvd = 0; + size_t m; + int attempts; + int redelivered = 0; + int i; + rd_kafka_message_t *first_msg = NULL; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_release_then_reject_no_redelivery ===\n"); + + rkshare = test_create_share_consumer(group, "explicit"); + topic = test_mk_topic_name("0172-release-reject", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 5); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume all messages */ + attempts = 30; + while (rcvd < 5 && attempts-- > 0 && batch_cnt < 30) { + rd_kafka_messages_t *b = NULL; + size_t batch_rcvd; + err = rd_kafka_share_poll(rkshare, 2000, &b); + if (err) + rd_kafka_error_destroy(err); + batch_rcvd = rd_kafka_messages_count(b); + if (batch_rcvd > 0) { + batches[batch_cnt++] = b; + rcvd += batch_rcvd; + } else { + rd_kafka_messages_destroy(b); + } + } + + TEST_ASSERT(rcvd == 5, "Expected 5 messages, got %zu", rcvd); + + /* Locate the first message across batches */ + first_msg = rd_kafka_messages_get(batches[0], 0); + + /* First RELEASE offset 0, then override with REJECT */ + ack_err = rd_kafka_share_acknowledge_type( + rkshare, first_msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, "RELEASE failed: %s", + rd_kafka_err2str(ack_err)); + + ack_err = rd_kafka_share_acknowledge_type( + rkshare, first_msg, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, "REJECT failed: %s", + rd_kafka_err2str(ack_err)); + + /* ACCEPT remaining messages - skip the very first one (index 0 of + * batches[0]) which is already REJECT'd above */ + { + rd_bool_t skipped_first = rd_false; + for (i = 0; i < batch_cnt; i++) { + size_t bcnt = rd_kafka_messages_count(batches[i]); + for (m = 0; m < bcnt; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batches[i], m); + if (!skipped_first) { + skipped_first = rd_true; + continue; + } + rd_kafka_share_acknowledge_type( + rkshare, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + } + } + } + + /* Destroy batches */ + for (i = 0; i < batch_cnt; i++) { + rd_kafka_messages_destroy(batches[i]); + batches[i] = NULL; + } + batch_cnt = 0; + + /* Poll for redelivery - should get 0 */ + TEST_SAY("Polling for redelivery (expecting 0)...\n"); + attempts = 5; + while (attempts-- > 0) { + size_t redeliv_rcvd; + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + redeliv_rcvd = rd_kafka_messages_count(batch); + + for (m = 0; m < redeliv_rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + redelivered++; + rd_kafka_share_acknowledge_type( + rkshare, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(redelivered == 0, + "Expected 0 redelivered (REJECT overrides RELEASE), got %d", + redelivered); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_release_then_reject_no_redelivery: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + + +/** + * @brief Test changing acknowledgement type before commit + * + * Verifies that acknowledgement type can be changed before commit completes. + */ +static void do_test_change_ack_type_before_commit(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + rd_kafka_resp_err_t ack_err; + rd_kafka_topic_partition_list_t *subs; + const char *group = "share-change-ack-type"; + const char *topic; + size_t rcvd = 0; + int attempts; + test_ack_cb_state_t state = {0}; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_change_ack_type_before_commit ===\n"); + + /* Explicit mode with explicit commit and callback */ + rkshare = + test_create_share_consumer_with_cb(group, "explicit", &state, NULL); + topic = test_mk_topic_name("0172-change-ack-explicit", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 1); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume one message */ + attempts = 30; + while (rcvd == 0 && attempts-- > 0) { + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + if (rcvd > 0) { + rd_kafka_share_acknowledge_type( + rkshare, rd_kafka_messages_get(batch, 0), + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + } + } + + TEST_SAY("Consumed %zu messages after first commit\n", rcvd); + + /* Second: Change to RELEASE for first message before commit */ + ack_err = rd_kafka_share_acknowledge_type( + rkshare, rd_kafka_messages_get(batch, 0), + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "RELEASE (changing from ACCEPT) failed: %s", + rd_kafka_err2str(ack_err)); + + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Commit async - should succeed with RELEASE as final ack type */ + err = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!err, "commit_async failed: %s", + err ? rd_kafka_error_string(err) : ""); + + /* Poll to receive the RELEASED message that should be redelivered + * This loop also processes the first callback */ + rcvd = 0; + attempts = 30; + while (rcvd == 0 && attempts-- > 0) { + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + if (rcvd > 0) { + rd_kafka_share_acknowledge_type( + rkshare, rd_kafka_messages_get(batch, 0), + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + + TEST_ASSERT(state.callback_cnt == 1, "Expected 1 callback, got %d", + state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&state) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "First callback failed: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + TEST_SAY("First commit callback received: offsets=%zu, err=%s\n", + state.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + TEST_SAY("Consumed %zu redelivered messages\n", rcvd); + TEST_ASSERT(rcvd == 1, "Expected 1 redelivered message, got %zu", rcvd); + + /* Commit async again for the remaining messages */ + err = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!err, "Second commit_async failed: %s", + err ? rd_kafka_error_string(err) : ""); + + /* Wait for second callback and verify (expecting 2 total callbacks) */ + test_wait_for_cb_with_poll(&state, rkshare, 2, 10000); + TEST_ASSERT(state.callback_cnt == 2, + "Expected 2 callbacks total, got %d", state.callback_cnt); + TEST_ASSERT(state.errs[1] == RD_KAFKA_RESP_ERR_NO_ERROR, + "Second callback failed: %s", + rd_kafka_err2name(state.errs[1])); + TEST_SAY("Second commit callback received: total_offsets=%zu, err=%s\n", + state.total_offsets, rd_kafka_err2name(state.errs[1])); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + TEST_SAY("=== test_change_ack_type_before_commit: PASSED ===\n"); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that acknowledging after commit fails + * + * After commit completes, acknowledged messages are removed from + * inflight_acks map. Attempting to acknowledge again should fail with _STATE. + */ +static void do_test_ack_after_commit(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + rd_kafka_resp_err_t ack_err; + rd_kafka_topic_partition_list_t *subs; + const char *group = "share-ack-after-commit"; + const char *topic; + size_t rcvd = 0; + int attempts; + const char *saved_topic = NULL; + int32_t saved_partition = -1; + int64_t saved_offset = -1; + test_ack_cb_state_t state = {0}; + rd_kafka_message_t *msg0; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_after_commit ===\n"); + + /* Test 1: Explicit mode with explicit commit and callback */ + rkshare = + test_create_share_consumer_with_cb(group, "explicit", &state, NULL); + topic = test_mk_topic_name("0172-ack-after-commit-explicit", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 1); + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume and acknowledge messages, save first message info */ + attempts = 30; + while (rcvd == 0 && attempts-- > 0) { + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + if (rcvd > 0) { + rd_kafka_share_acknowledge_type( + rkshare, rd_kafka_messages_get(batch, 0), + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + } + } + + TEST_ASSERT(rcvd == 1, "Expected 1 message, got %zu", rcvd); + + /* Save first message info and acknowledge all */ + msg0 = rd_kafka_messages_get(batch, 0); + saved_topic = topic; + saved_partition = msg0->partition; + saved_offset = msg0->offset; + + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Explicit commit */ + err = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!err, "commit_async failed"); + + /* Wait for callback and verify */ + test_wait_for_cb_with_poll(&state, rkshare, 1, 10000); + TEST_ASSERT(state.callback_cnt >= 1, + "Expected at least 1 callback, got %d", state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&state) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Callback failed: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + TEST_SAY("Commit callback received: offsets=%zu, err=%s\n", + state.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + + /* Now try to acknowledge the same message again - should fail */ + ack_err = rd_kafka_share_acknowledge_offset( + rkshare, saved_topic, saved_partition, saved_offset, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE after commit, got %s", + rd_kafka_err2str(ack_err)); + TEST_SAY( + "Explicit mode: Re-ack after commit correctly failed with " + "_STATE\n"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + TEST_SAY("=== test_ack_after_commit: PASSED ===\n"); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Max Delivery Attempts Tests + ***************************************************************************/ + +/** + * @brief Release a record 5 times (max attempts), verify no 6th delivery + * + * Default share.record.lock.partition.limit is 5. After 5 RELEASE attempts, + * the broker should not attempt any redelivery. + */ +static void do_test_max_delivery_attempts(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + rd_kafka_topic_partition_list_t *subs; + const char *group = "share-max-delivery-attempts"; + const char *topic; + size_t rcvd; + int delivery_attempt; + int attempts; + const int max_deliveries = 5; + rd_kafka_message_t *msg0; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_max_delivery_attempts ===\n"); + TEST_SAY( + "Testing that record is not redelivered after %d RELEASE " + "attempts\n", + max_deliveries); + + rkshare = test_create_share_consumer(group, "explicit"); + topic = test_mk_topic_name("0172-max-delivery", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, + 1); /* Just 1 message */ + + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* RELEASE the same message max_deliveries times */ + for (delivery_attempt = 1; delivery_attempt <= max_deliveries; + delivery_attempt++) { + rcvd = 0; + attempts = 30; + + while (rcvd == 0 && attempts-- > 0) { + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + rcvd = rd_kafka_messages_count(batch); + } + + TEST_ASSERT(rcvd == 1, + "Delivery attempt %d: expected 1 message, got %zu", + delivery_attempt, rcvd); + + msg0 = rd_kafka_messages_get(batch, 0); + + /* Verify delivery_count matches attempt number */ + TEST_ASSERT( + rd_kafka_message_delivery_count(msg0) == delivery_attempt, + "Delivery attempt %d: expected delivery_count=%d, got %d", + delivery_attempt, delivery_attempt, + rd_kafka_message_delivery_count(msg0)); + + TEST_SAY( + "Delivery attempt %d: received message " + "(delivery_count=%d), sending RELEASE\n", + delivery_attempt, rd_kafka_message_delivery_count(msg0)); + + /* RELEASE to trigger redelivery */ + rd_kafka_share_acknowledge_type( + rkshare, msg0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Now poll again - message should NOT be redelivered (max attempts + * reached) */ + TEST_SAY("Polling for 6th delivery (should NOT receive message)...\n"); + rcvd = 0; + attempts = 5; + while (attempts-- > 0) { + size_t batch_rcvd; + err = rd_kafka_share_poll(rkshare, 2000, &batch); + if (err) + rd_kafka_error_destroy(err); + batch_rcvd = rd_kafka_messages_count(batch); + + if (batch_rcvd > 0) { + size_t m; + for (m = 0; m < batch_rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + rcvd++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(rcvd == 0, + "Expected 0 messages after %d RELEASE attempts, got %zu", + max_deliveries, rcvd); + + TEST_SAY("SUCCESS: Message not redelivered after %d RELEASE attempts\n", + max_deliveries); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_max_delivery_attempts: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + +/*************************************************************************** + * High-Intensity Random Acknowledgement Tests + * + * These tests produce large numbers of messages and apply random + * acknowledgement types (ACCEPT/REJECT/RELEASE) to stress test the + * acknowledgement infrastructure across different topologies. + ***************************************************************************/ + +/** + * @brief High-intensity random acks: Single topic, single partition + * + * 5000 messages on 1 topic with 1 partition. + * Random ACCEPT/REJECT/RELEASE for each message. + */ +static void do_test_random_ack_single_topic_single_partition(void) { + ack_test_config_t config = {.test_name = "random-ack-1t-1p-5000msgs", + .topic_cnt = 1, + .partitions = {1}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 5000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief High-intensity random acks: Multiple topics, single partition each + * + * 5000 messages across 4 topics, 1 partition each (~1250 msgs per topic). + * Random ACCEPT/REJECT/RELEASE for each message. + */ +static void do_test_random_ack_multiple_topics_single_partition(void) { + ack_test_config_t config = {.test_name = "random-ack-4t-1p-5000msgs", + .topic_cnt = 4, + .partitions = {1, 1, 1, 1}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 5000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief High-intensity random acks: Single topic, multiple partitions + * + * 5000 messages on 1 topic with 4 partitions (~1250 msgs per partition). + * Random ACCEPT/REJECT/RELEASE for each message. + */ +static void do_test_random_ack_single_topic_multiple_partitions(void) { + ack_test_config_t config = {.test_name = "random-ack-1t-4p-5000msgs", + .topic_cnt = 1, + .partitions = {4}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 5000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief High-intensity random acks: Multiple topics, multiple partitions + * + * 5000 messages across 2 topics, 2 partitions each (~1250 msgs per partition). + * Random ACCEPT/REJECT/RELEASE for each message. + */ +static void do_test_random_ack_multiple_topics_multiple_partitions(void) { + ack_test_config_t config = {.test_name = "random-ack-2t-2p-5000msgs", + .topic_cnt = 2, + .partitions = {2, 2}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 5000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Scale test: 15 topics, 1 partition each, 10000 messages + * + * ~666 messages per topic. Tests handling many topics with random acks. + */ +static void do_test_scale_15_topics_single_partition(void) { + ack_test_config_t config = { + .test_name = "scale-15t-1p-10000msgs", + .topic_cnt = 15, + .partitions = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 10000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Scale test: 15 topics, 2 partitions each, 10000 messages + * + * ~333 messages per partition (30 partitions total). + * Tests handling many topics with multiple partitions. + */ +static void do_test_scale_15_topics_multiple_partitions(void) { + ack_test_config_t config = { + .test_name = "scale-15t-2p-10000msgs", + .topic_cnt = 15, + .partitions = {2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 10000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Scale test: 8 topics, 4 partitions each, 8000 messages + * + * ~250 messages per partition (32 partitions total). + * Tests high partition count scenario. + */ +static void do_test_scale_8_topics_4_partitions(void) { + ack_test_config_t config = {.test_name = "scale-8t-4p-8000msgs", + .topic_cnt = 8, + .partitions = {4, 4, 4, 4, 4, 4, 4, 4}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 8000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Scale test: Single topic, 8 partitions, 10000 messages + * + * 1250 messages per partition. Tests single topic with many partitions. + */ +static void do_test_scale_single_topic_8_partitions(void) { + ack_test_config_t config = {.test_name = "scale-1t-8p-10000msgs", + .topic_cnt = 1, + .partitions = {8}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 10000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + +/** + * @brief Scale test: 10 topics, 3 partitions each, 15000 messages + * + * 500 messages per partition (30 partitions total). + * Large scale test for acknowledgement infrastructure. + */ +static void do_test_scale_10_topics_3_partitions(void) { + ack_test_config_t config = { + .test_name = "scale-10t-3p-15000msgs", + .topic_cnt = 10, + .partitions = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3}, + .consumer_cnt = 1, + .use_random_acks = rd_true, + .total_msgs = 15000}; + SUB_TEST(); + run_ack_test(&config); + SUB_TEST_PASS(); +} + + +/** + * @brief Re-acknowledging a record after the batch has been polled past + * returns _STATE. + * + * In explicit mode, poll batch1 (1 record), acknowledge it, poll batch2 + * (empty), then try to acknowledge batch1's record again. The second + * acknowledge must return _STATE because the record is no longer in the + * current batch / inflight map. + */ +static void do_test_ack_message_from_earlier_batch(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch1 = NULL; + rd_kafka_messages_t *batch2 = NULL; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t ack_err; + const char *group = "share-cross-batch-ack"; + const char *topic; + size_t b1_rcvd = 0; + size_t r = 0; + rd_kafka_error_t *e; + int attempts; + rd_kafka_message_t *b1_msg0; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_message_from_earlier_batch ===\n"); + + rkshare = test_create_share_consumer(group, "explicit"); + topic = test_mk_topic_name("0172-cross-batch", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 1); + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Poll batch 1 */ + attempts = 30; + while (b1_rcvd < 1 && attempts-- > 0) { + if (batch1) { + rd_kafka_messages_destroy(batch1); + batch1 = NULL; + } + e = rd_kafka_share_poll(rkshare, 2000, &batch1); + if (e) + rd_kafka_error_destroy(e); + b1_rcvd = rd_kafka_messages_count(batch1); + } + TEST_ASSERT(b1_rcvd == 1, "Expected 1 record in batch1, got %zu", + b1_rcvd); + + b1_msg0 = rd_kafka_messages_get(batch1, 0); + + ack_err = rd_kafka_share_acknowledge(rkshare, b1_msg0); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "First ack failed: %s", rd_kafka_err2name(ack_err)); + + /* Poll batch 2 - must be empty (we already consumed and acked + * the only record). */ + e = rd_kafka_share_poll(rkshare, 2000, &batch2); + if (e) + rd_kafka_error_destroy(e); + r = rd_kafka_messages_count(batch2); + TEST_ASSERT(r == 0, + "Expected batch 2 to be empty after acking the only " + "record, got %zu records", + r); + rd_kafka_messages_destroy(batch2); + batch2 = NULL; + + /* Re-acknowledge from batch1 - should fail with _STATE */ + ack_err = rd_kafka_share_acknowledge(rkshare, b1_msg0); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE for re-ack across batches, got %s", + rd_kafka_err2name(ack_err)); + + rd_kafka_messages_destroy(batch1); + batch1 = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_ack_message_from_earlier_batch: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief acknowledge_offset() before any record has been consumed + * returns _STATE. + * + * Subscribe but do not poll; call acknowledge_offset() with arbitrary + * topic/partition/offset values and expect _STATE (nothing in inflight). + */ +static void do_test_ack_offset_before_consume(void) { + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + const char *group = "share-ack-before-consume"; + const char *topic; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_offset_before_consume ===\n"); + + rkshare = test_create_share_consumer(group, "explicit"); + topic = test_mk_topic_name("0172-ack-precons", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, 0, 0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE for ack_offset before any consume, " + "got %s", + rd_kafka_err2name(err)); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_ack_offset_before_consume: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief acknowledge_offset() with wrong topic / partition / offset + * returns _STATE; correct values succeed. + * + * Consume one record so we have a valid (topic, partition, offset) tuple + * in the inflight map. Calling acknowledge_offset() with wrong values for + * any of the three components must return _STATE. The correct values then + * succeed. + */ +static void do_test_ack_offset_wrong_params(void) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + const char *group = "share-ack-wrong-params"; + const char *topic; + size_t rcvd = 0; + int attempts; + int32_t partition; + int64_t offset; + rd_kafka_message_t *msg0; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY("=== test_ack_offset_wrong_params ===\n"); + + rkshare = test_create_share_consumer(group, "explicit"); + topic = test_mk_topic_name("0172-wrong-params", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 1); + test_share_set_auto_offset_reset(group, "earliest"); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + attempts = 30; + while (rcvd < 1 && attempts-- > 0) { + rd_kafka_error_t *e; + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + e = rd_kafka_share_poll(rkshare, 2000, &batch); + if (e) + rd_kafka_error_destroy(e); + rcvd = rd_kafka_messages_count(batch); + } + TEST_ASSERT(rcvd == 1, "Expected 1 record, got %zu", rcvd); + + msg0 = rd_kafka_messages_get(batch, 0); + partition = msg0->partition; + offset = msg0->offset; + + /* Wrong offset */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, partition, offset + 100, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE for wrong offset, got %s", + rd_kafka_err2name(err)); + + /* Wrong topic */ + err = rd_kafka_share_acknowledge_offset( + rkshare, "0172-nonexistent-topic", partition, offset, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE for wrong topic, got %s", + rd_kafka_err2name(err)); + + /* Wrong partition */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, partition + 100, offset, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE for wrong partition, got %s", + rd_kafka_err2name(err)); + + /* Correct values - should succeed */ + err = rd_kafka_share_acknowledge_offset( + rkshare, topic, partition, offset, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected success for correct params, got %s", + rd_kafka_err2name(err)); + + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_SAY("=== test_ack_offset_wrong_params: PASSED ===\n"); + + SUB_TEST_PASS(); +} + + +/** + * @brief Mix of implicit and explicit ack-mode consumers in the same group. + * + * `share.acknowledgement.mode` is a CLIENT-side config, so two consumers + * in the same share group can use different modes. Verifies that: + * - The implicit consumer auto-acks on each consume_batch. + * - The explicit consumer's accepted records are not redelivered after + * commit_sync. + * - All produced records are delivered (no losses), and the total + * count matches the produced count once the explicit consumer + * commits (no duplicates). + */ +static void do_test_mixed_ack_mode_same_group(void) { + const char *group = "share-mixed-ack-mode"; + const char *topic; + rd_kafka_share_t *implicit_c; + rd_kafka_share_t *explicit_c; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + const int total_msgs = 400; + int implicit_cnt = 0; + int explicit_cnt = 0; + int attempts; + rd_kafka_error_t *cerr; + test_ack_cb_state_t exp_state = {0}; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "=== Mixed ack-mode (implicit + explicit) in same group ===\n"); + + topic = test_mk_topic_name("0172-mixed-ack-mode", 1); + test_create_topic_wait_exists(NULL, topic, 4, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, RD_KAFKA_PARTITION_UA, + total_msgs); + + test_share_set_auto_offset_reset(group, "earliest"); + + implicit_c = test_create_share_consumer(group, "implicit"); + explicit_c = test_create_share_consumer_with_cb(group, "explicit", + &exp_state, NULL); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(implicit_c, subs); + rd_kafka_share_subscribe(explicit_c, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* Round-robin poll across both consumers. The implicit consumer's + * messages are auto-acked by the next consume_batch call; the + * explicit consumer must call rd_kafka_share_acknowledge per + * message. */ + attempts = 200; + while (implicit_cnt + explicit_cnt < total_msgs && attempts-- > 0) { + size_t rcvd = 0; + size_t m; + rd_kafka_error_t *err; + + /* Implicit consumer */ + err = rd_kafka_share_poll(implicit_c, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + } else { + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) + implicit_cnt++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Explicit consumer */ + rcvd = 0; + err = rd_kafka_share_poll(explicit_c, 1000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (!msg->err) { + rd_kafka_resp_err_t ack_err = + rd_kafka_share_acknowledge_type( + explicit_c, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(!ack_err, + "explicit ACCEPT failed: %s", + rd_kafka_err2str(ack_err)); + explicit_cnt++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumed: implicit=%d explicit=%d total=%d/%d\n", + implicit_cnt, explicit_cnt, implicit_cnt + explicit_cnt, + total_msgs); + + /* Flush explicit consumer's accepted acks to the broker so they + * are durable before we tear the consumer down. */ + rd_kafka_topic_partition_list_t *commit_parts = NULL; + cerr = rd_kafka_share_commit_sync(explicit_c, 30000, &commit_parts); + TEST_ASSERT(!cerr, "explicit commit_sync failed: %s", + cerr ? rd_kafka_error_string(cerr) : ""); + if (commit_parts) + rd_kafka_topic_partition_list_destroy(commit_parts); + TEST_ASSERT(test_ack_cb_state_first_err(&exp_state) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "explicit ack callback reported err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&exp_state))); + + /* Final flush poll on the implicit consumer so the last batch's + * piggybacked acks reach the broker. */ + rd_kafka_error_t *err = rd_kafka_share_poll(implicit_c, 1000, &batch); + if (err) + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(implicit_cnt + explicit_cnt == total_msgs, + "Expected exactly %d total records across both consumers, " + "got %d (implicit=%d explicit=%d)", + total_msgs, implicit_cnt + explicit_cnt, implicit_cnt, + explicit_cnt); + TEST_ASSERT(implicit_cnt > 0, "Implicit consumer received no records"); + TEST_ASSERT(explicit_cnt > 0, "Explicit consumer received no records"); + + test_share_consumer_close(implicit_c); + test_share_destroy(implicit_c); + test_share_consumer_close(explicit_c); + test_share_destroy(explicit_c); + test_ack_cb_state_destroy(&exp_state); + + TEST_SAY( + "SUCCESS: mixed ack-mode group - implicit=%d explicit=%d " + "(total=%d)\n", + implicit_cnt, explicit_cnt, implicit_cnt + explicit_cnt); + + SUB_TEST_PASS(); +} + + +int main_0172_share_consumer_acknowledge(int argc, char **argv) { + + test_timeout_set(600); /* 10 minutes for all tests */ + + /* Create common handles for all tests */ + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + /* Core tests */ + do_test_release_redelivery(); + do_test_reject_no_redelivery(); + do_test_accept_no_redelivery(); + + /* Error handling tests */ + do_test_ack_null_message(); + do_test_ack_null_rkshare(); + do_test_ack_invalid_type(); + do_test_release_then_reject_no_redelivery(); + + /* Acknowledgement state tests */ + do_test_change_ack_type_before_commit(); + do_test_ack_after_commit(); + do_test_ack_message_from_earlier_batch(); + do_test_ack_offset_before_consume(); + do_test_ack_offset_wrong_params(); + + /* Max delivery attempts test */ + do_test_max_delivery_attempts(); + + /* High-intensity random acknowledgement tests */ + do_test_random_ack_single_topic_single_partition(); + do_test_random_ack_multiple_topics_single_partition(); + do_test_random_ack_single_topic_multiple_partitions(); + do_test_random_ack_multiple_topics_multiple_partitions(); + + /* Scale tests - high topic/partition counts with many messages */ + do_test_scale_15_topics_single_partition(); + do_test_scale_15_topics_multiple_partitions(); + do_test_scale_8_topics_4_partitions(); + do_test_scale_single_topic_8_partitions(); + do_test_scale_10_topics_3_partitions(); + + /* Mixed ack-mode within the same share group */ + do_test_mixed_ack_mode_same_group(); + + /* Cleanup common handles */ + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0173-share_consumer_commit_async.c b/lib/librdkafka-2.15.0/tests/0173-share_consumer_commit_async.c new file mode 100644 index 00000000000..165913d99b4 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0173-share_consumer_commit_async.c @@ -0,0 +1,2735 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @brief Share consumer rd_kafka_share_commit_async() API tests. + * + * Tests the commit_async API in both implicit and explicit ack modes. + * Verifies that commit_async commits acknowledged records so they are + * not redelivered, and that it properly extracts acks from the inflight + * map (preventing consume_batch from re-sending the same acks). + * + * Mixed ack types: ACCEPT (~50%), RELEASE (~40%), REJECT (~10%) + */ + +#define MAX_MSGS 500 +#define CONSUME_ARRAY 10001 + +/** Common producer reused across all non-mock subtests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all non-mock subtests. */ +static rd_kafka_t *common_admin; + + +/** + * @brief Set share.record.lock.duration.ms for a share group. + */ +static void set_group_lock_duration(const char *group_name, + const char *duration_ms) { + const char *cfg[] = {"share.record.lock.duration.ms", "SET", + duration_ms}; + + test_IncrementalAlterConfigs_simple( + common_admin, RD_KAFKA_RESOURCE_GROUP, group_name, cfg, 1); +} + + +/** + * @brief Subscribe a share consumer to topics. + */ +static void subscribe_consumer(rd_kafka_share_t *rkshare, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + int i; + + subs = rd_kafka_topic_partition_list_new(topic_cnt); + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(subs, topics[i], + RD_KAFKA_PARTITION_UA); + + err = rd_kafka_share_subscribe(rkshare, subs); + TEST_ASSERT(!err, "Subscribe failed: %s", rd_kafka_err2str(err)); + + rd_kafka_topic_partition_list_destroy(subs); +} + + + +/** + * @brief Determine random ack type with distribution: + * ACCEPT ~50%, RELEASE ~40%, REJECT ~10% + */ +static rd_kafka_share_AcknowledgeType_t get_random_ack_type(void) { + int r = jitter(0, 99); + if (r < 50) + return RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + else if (r < 90) + return RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE; + else + return RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT; +} + + +/* =================================================================== + * Implicit ack mode — commit_async commits acks independently of + * consume_batch piggybacking. + * + * Consumer 1 (implicit) consumes first batch, calls commit_async, + * waits, closes. Consumer 2 consumes remaining and verifies no + * offset overlap with consumer 1. + * =================================================================== */ +static void do_test_implicit_second_consumer(void) { + const char *topic; + const char *group = "commit-async-implicit-second"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed1 = 0, consumed2 = 0; + int attempts = 0; + int64_t *c1_offsets; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-impl-2nd", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, MAX_MSGS); + + rkshare = test_create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + c1_offsets = rd_calloc(MAX_MSGS, sizeof(*c1_offsets)); + + /* Wait for first batch of records */ + while (consumed1 == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + c1_offsets[consumed1++] = rkm->offset; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumer 1 consumed %d messages in first batch\n", consumed1); + TEST_ASSERT(consumed1 > 0, "Consumer 1 got no records"); + + /* commit_async without any subsequent consume_batch */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Second consumer: should only get the 5 verification records. + * No lock wait needed — implicit mode close tears down the + * connection and broker releases records immediately. */ + rkshare = test_create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer 2 consume_batch returned: rcvd=%zu, error=%s\n", + rcvd, error ? rd_kafka_error_string(error) : "none"); + if (error) { + rd_kafka_error_destroy(error); + } + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer 2 got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed2++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer 2 got %d messages (expected 5)\n", consumed2); + TEST_ASSERT(consumed2 == 5, "Expected 5 verification records, got %d", + consumed2); + + rd_free(c1_offsets); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Explicit ack mode — commit_async commits explicit ACCEPT acks. + * + * Consumer 1 (explicit) consumes first batch, ACCEPTs each record, + * calls commit_async, waits, closes. Consumer 2 consumes remaining + * and verifies no offset overlap with consumer 1. + * =================================================================== */ +static void do_test_explicit_second_consumer(void) { + const char *topic; + const char *group = "commit-async-explicit-second"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed1 = 0, consumed2 = 0; + int attempts = 0; + int64_t *c1_offsets; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-expl-2nd", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, MAX_MSGS); + + rkshare = test_create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + c1_offsets = rd_calloc(MAX_MSGS, sizeof(*c1_offsets)); + + /* Wait for first batch of records */ + while (consumed1 == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + c1_offsets[consumed1++] = rkm->offset; + rd_kafka_share_acknowledge(rkshare, rkm); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumer 1 consumed %d messages in first batch\n", consumed1); + TEST_ASSERT(consumed1 > 0, "Consumer 1 got no records"); + + /* commit_async without any subsequent consume_batch */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Second consumer: should only get the 5 verification records. + * Records are either committed by the last commit_async or + * released on the broker side when the connection is closed. */ + rkshare = test_create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer 2 consume_batch returned: rcvd=%zu, error=%s\n", + rcvd, error ? rd_kafka_error_string(error) : "none"); + if (error) { + rd_kafka_error_destroy(error); + } + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer 2 got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed2++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer 2 got %d messages (expected 5)\n", consumed2); + TEST_ASSERT(consumed2 == 5, "Expected 5 verification records, got %d", + consumed2); + + rd_free(c1_offsets); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Mixed ack types — commit_async with ACCEPT/RELEASE/REJECT. + * + * Consumer 1 (explicit) acks each record randomly (ACCEPT ~50%, + * RELEASE ~40%, REJECT ~10%), calls commit_async after each batch. + * Redeliveries (delivery_count > 1) are ACCEPT'd. Drain loop + * flushes remaining redeliveries. Consumer 2 gets unredelivered + * RELEASE'd records. + * =================================================================== */ +static void do_test_mixed_acks_second_consumer(void) { + const char *topic; + const char *group = "commit-async-mixed-second"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0, redelivered = 0; + int released_cnt = 0; + int attempts = 0; + int64_t *released_offsets; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-mixed-2nd", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, MAX_MSGS); + + rkshare = test_create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + released_offsets = rd_calloc(MAX_MSGS, sizeof(*released_offsets)); + + /* Consume all records and handle redeliveries in the same loop */ + while ((consumed < MAX_MSGS || redelivered < released_cnt) && + attempts++ < 100) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + if (rd_kafka_message_delivery_count(rkm) > 1) { + /* Redelivered — verify it was + * RELEASE'd and ACCEPT it */ + int k; + rd_bool_t found = rd_false; + + for (k = 0; k < released_cnt; k++) { + if (rkm->offset == + released_offsets[k]) { + found = rd_true; + released_offsets[k] = + -1; + break; + } + } + TEST_ASSERT( + found, + "Redelivered offset %" PRId64 + " was not RELEASE'd", + rkm->offset); + + rd_kafka_share_acknowledge(rkshare, + rkm); + redelivered++; + } else { + /* New record — ack with random type */ + rd_kafka_share_AcknowledgeType_t + ack_type = get_random_ack_type(); + + if (ack_type == + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT) { + rd_kafka_share_acknowledge( + rkshare, rkm); + } else { + if (ack_type == + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE) + released_offsets + [released_cnt++] = + rkm->offset; + rd_kafka_share_acknowledge_type( + rkshare, rkm, ack_type); + } + consumed++; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + } + + TEST_SAY("Consumed %d/%d, released %d, redelivered %d/%d\n", consumed, + MAX_MSGS, released_cnt, redelivered, released_cnt); + TEST_ASSERT(consumed == MAX_MSGS, "Expected %d consumed, got %d", + MAX_MSGS, consumed); + TEST_ASSERT(redelivered == released_cnt, + "Expected %d redelivered, got %d", released_cnt, + redelivered); + + rd_free(released_offsets); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + + +/* =================================================================== + * Multiple topics x partitions — commit_async across many + * topic-partitions. + * + * Creates 10 topics x 5 partitions. Runs 20 rounds: each round + * produces messages, consumes them, and calls commit_async. + * =================================================================== */ +static void do_test_multi_topic_partition(void) { + const int topic_cnt = 10; + const int part_cnt = 5; + const int total_parts = topic_cnt * part_cnt; + const int rounds = 20; + const int msgs_per_part = 10; + const int msgs_per_round = msgs_per_part * total_parts; + const char *group = "commit-async-multi-tp"; + const char *topics[10]; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + int t, p, round; + int total_consumed = 0; + + SUB_TEST(); + + for (t = 0; t < topic_cnt; t++) { + topics[t] = + rd_strdup(test_mk_topic_name("0173-ca-multi-tp", 1)); + test_create_topic_wait_exists(NULL, topics[t], part_cnt, -1, + 60 * 1000); + } + + rkshare = test_create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, topics, topic_cnt); + + for (round = 0; round < rounds; round++) { + int consumed = 0; + int attempts = 0; + + for (t = 0; t < topic_cnt; t++) { + for (p = 0; p < part_cnt; p++) + test_produce_msgs_simple(common_producer, + topics[t], p, + msgs_per_part); + } + + while (consumed < msgs_per_round && attempts++ < 100) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 5000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + } + + TEST_SAY("Round %d: consumed %d/%d messages\n", round, consumed, + msgs_per_round); + TEST_ASSERT(consumed == msgs_per_round, + "Round %d: Expected %d, got %d", round, + msgs_per_round, consumed); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "Round %d: commit_async failed: %s", round, + error ? rd_kafka_error_string(error) : ""); + + total_consumed += consumed; + } + + TEST_SAY("Total consumed across %d rounds: %d\n", rounds, + total_consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + for (t = 0; t < topic_cnt; t++) + rd_free((void *)topics[t]); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Produce-consume loop — mixing commit_async and consume_batch + * piggybacking. + * + * 5 rounds: each round produces N, consumes N. Even rounds use + * commit_async, odd rounds let consume_batch handle acks on the + * next poll. + * =================================================================== */ +static void do_test_produce_consume_loop(void) { + const char *topic; + const char *group = "commit-async-loop"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + int round; + const int rounds = 5; + const int msgs_per_round = MAX_MSGS / rounds; + int total_consumed = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-loop", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + rkshare = test_create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + for (round = 0; round < rounds; round++) { + int consumed = 0; + int attempts = 0; + + test_produce_msgs_simple(common_producer, topic, 0, + msgs_per_round); + TEST_SAY("Round %d: produced %d messages\n", round, + msgs_per_round); + + while (consumed < msgs_per_round && attempts++ < 100) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 5000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + } + + TEST_SAY("Round %d: consumed %d/%d messages\n", round, consumed, + msgs_per_round); + TEST_ASSERT(consumed == msgs_per_round, + "Round %d: Expected %d, got %d", round, + msgs_per_round, consumed); + + if (round % 2 == 0) { + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "Round %d: commit_async failed: %s", + round, + error ? rd_kafka_error_string(error) : ""); + TEST_SAY("Round %d: used commit_async\n", round); + } else { + TEST_SAY( + "Round %d: letting consume_batch handle acks\n", + round); + } + + total_consumed += consumed; + } + + TEST_SAY("Total consumed across %d rounds: %d\n", rounds, + total_consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Multi-round mixed acks — per-round produce, consume, and + * commit_async with same-consumer redelivery handling. + * + * 3 rounds: each round produces msgs_per_round, consumes them, + * acks randomly (ACCEPT/RELEASE/REJECT), calls commit_async. + * Redeliveries (delivery_count > 1) are ACCEPT'd inline within + * the same round. Verifies consumed + redelivered counts per round. + * =================================================================== */ +static void do_test_multi_round_mixed_second_consumer(void) { + const char *topic; + const char *group = "commit-async-multi-round-second"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + int round; + const int rounds = 3; + const int msgs_per_round = MAX_MSGS / rounds; + int total_consumed = 0; + int total_released = 0; + int total_redelivered = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-mr-2nd", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + rkshare = test_create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + for (round = 0; round < rounds; round++) { + test_produce_msgs_simple(common_producer, topic, 0, + msgs_per_round); + int consumed = 0; + int released_cnt = 0; + int redelivered = 0; + int attempts = 0; + + /* Consume msgs_per_round new records, handling redeliveries + * from previous rounds/batches inline */ + while ( + (consumed < msgs_per_round || redelivered < released_cnt) && + attempts++ < 100) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 5000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + if (rd_kafka_message_delivery_count( + rkm) > 1) { + rd_kafka_share_acknowledge( + rkshare, rkm); + redelivered++; + } else { + rd_kafka_share_AcknowledgeType_t + ack_type = + get_random_ack_type(); + + if (ack_type == + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT) { + rd_kafka_share_acknowledge( + rkshare, rkm); + } else { + if (ack_type == + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE) + released_cnt++; + rd_kafka_share_acknowledge_type( + rkshare, rkm, + ack_type); + } + consumed++; + } + } + } + rd_kafka_messages_destroy(batch); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "Round %d: commit_async failed: %s", + round, + error ? rd_kafka_error_string(error) : ""); + } + + TEST_SAY( + "Round %d: consumed %d/%d, released %d, " + "redelivered %d/%d\n", + round, consumed, msgs_per_round, released_cnt, redelivered, + released_cnt); + TEST_ASSERT(consumed == msgs_per_round, + "Round %d: expected %d consumed, got %d", round, + msgs_per_round, consumed); + TEST_ASSERT(redelivered == released_cnt, + "Round %d: expected %d redelivered, got %d", round, + released_cnt, redelivered); + + total_consumed += consumed; + total_released += released_cnt; + total_redelivered += redelivered; + } + + TEST_SAY("Total: consumed %d, released %d, redelivered %d\n", + total_consumed, total_released, total_redelivered); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * commit_async with no pending acks — returns NULL (no error). + * =================================================================== */ +static void do_test_no_pending_acks(void) { + const char *group = "commit-async-no-pending"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + + SUB_TEST(); + + rkshare = test_create_share_consumer(group, "implicit"); + + error = rd_kafka_share_commit_async(rkshare); + TEST_SAY("commit_async with no acks: error=%s\n", + error ? rd_kafka_error_string(error) : "NULL"); + TEST_ASSERT(!error, "Expected NULL when no pending acks, got error: %s", + error ? rd_kafka_error_string(error) : ""); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Multiple consecutive commit_async calls — verifies no ack + * duplication or loss. + * + * Produce first set, consume, produce second set, call commit_async + * 3 times, then consume and verify count equals second produce amount. + * =================================================================== */ +static void do_test_multiple_commit_async_calls(void) { + const char *topic; + const char *group = "commit-async-multiple-calls"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + const int first_produce = MAX_MSGS / 2; + const int second_produce = MAX_MSGS / 2; + int consumed = 0, consumed2 = 0, call; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-multi-call", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + rkshare = test_create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + test_produce_msgs_simple(common_producer, topic, 0, first_produce); + + while (consumed < first_produce && attempts++ < 100) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + } + + TEST_SAY("First set: consumed %d/%d messages\n", consumed, + first_produce); + TEST_ASSERT(consumed == first_produce, "Expected %d, got %d", + first_produce, consumed); + + test_produce_msgs_simple(common_producer, topic, 0, second_produce); + + for (call = 0; call < 10; call++) { + error = rd_kafka_share_commit_async(rkshare); + TEST_SAY("commit_async call %d: error=%s\n", call, + error ? rd_kafka_error_string(error) : "NULL"); + TEST_ASSERT(!error, "commit_async call %d failed: %s", call, + error ? rd_kafka_error_string(error) : ""); + } + + attempts = 0; + while (consumed2 < second_produce && attempts++ < 100) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed2++; + } + rd_kafka_messages_destroy(batch); + } + + TEST_SAY("After multiple commits: consumed %d/%d messages\n", consumed2, + second_produce); + TEST_ASSERT(consumed2 == second_produce, + "Expected %d (second produce), got %d", second_produce, + consumed2); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * commit_async between produce rounds — correctly separates acks. + * + * Produce N, consume N, commit_async, wait for lock timeout, + * produce N more, consume N, commit_async. Second consumer verifies + * nothing left. + * =================================================================== */ +static void do_test_commit_between_produces(void) { + const char *topic; + const char *group = "commit-async-between-produces"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + const int half = MAX_MSGS / 2; + int consumed1 = 0, consumed2 = 0, received = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-between", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + rkshare = test_create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + /* First half: produce, wait for records, commit_async */ + test_produce_msgs_simple(common_producer, topic, 0, half); + + while (consumed1 == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT( + rd_kafka_message_delivery_count(rkm) == 1, + "First half: redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed1++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("First half: consumed %d messages\n", consumed1); + TEST_ASSERT(consumed1 == half, "First half: expected %d, got %d", half, + consumed1); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Wait for acquisition lock timeout (3 s + buffer) so first half's acks + * are fully committed or released before producing the second + * half */ + rd_sleep(4); + + /* Second half: produce more, wait for records, commit_async */ + test_produce_msgs_simple(common_producer, topic, 0, half); + + attempts = 0; + while (consumed2 == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT( + rd_kafka_message_delivery_count(rkm) == 1, + "Second half: redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed2++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Second half: consumed %d messages\n", consumed2); + TEST_ASSERT(consumed2 == half, "Second half: expected %d, got %d", half, + consumed2); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Second consumer: should only get the 5 verification records. + * No lock wait needed — implicit mode close tears down the + * connection and broker releases records immediately. */ + rkshare = test_create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer 2 consume_batch returned: rcvd=%zu, error=%s\n", + rcvd, error ? rd_kafka_error_string(error) : "none"); + if (error) { + rd_kafka_error_destroy(error); + } + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer 2 got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + received++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer 2 got %d messages (expected 5)\n", received); + TEST_ASSERT(received == 5, "Expected 5 verification records, got %d", + received); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * All RELEASE — commit_async with all-RELEASE acks causes full + * redelivery within the same consumer. + * + * Consumer (explicit) RELEASEs every new record and ACCEPT's + * redeliveries (delivery_count > 1). Verifies all original records + * are consumed and all redeliveries are received. + * =================================================================== */ +static void do_test_all_release_second_consumer(void) { + const char *topic; + const char *group = "commit-async-all-release-second"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + int consumed = 0, redelivered = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-allrel-2nd", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, MAX_MSGS); + + rkshare = test_create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume all records: RELEASE new records, ACCEPT redeliveries. + * Loop until all original records are consumed AND all + * redeliveries are received. */ + while ((consumed < MAX_MSGS || redelivered < consumed) && + attempts++ < 200) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + if (rd_kafka_message_delivery_count(rkm) > 1) { + rd_kafka_share_acknowledge(rkshare, + rkm); + redelivered++; + } else { + rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + consumed++; + } + } + } + rd_kafka_messages_destroy(batch); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + } + + TEST_SAY("Consumed %d/%d, redelivered %d/%d\n", consumed, MAX_MSGS, + redelivered, consumed); + TEST_ASSERT(consumed == MAX_MSGS, "Expected %d consumed, got %d", + MAX_MSGS, consumed); + TEST_ASSERT(redelivered == consumed, "Expected %d redelivered, got %d", + consumed, redelivered); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * All REJECT — commit_async with all-REJECT acks. Records are not + * redelivered. + * + * Consumer 1 (explicit) REJECTs every record, calls commit_async. + * Consumer 2 verifies 0 messages (REJECT'd records are archived). + * =================================================================== */ +static void do_test_all_reject_second_consumer(void) { + const char *topic; + const char *group = "commit-async-all-reject-second"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0, received = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-allrej-2nd", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, MAX_MSGS); + + rkshare = test_create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed < MAX_MSGS && attempts++ < 100) { + size_t rcvd_inner; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd_inner = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd_inner; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + } + + TEST_SAY("Consumer 1 consumed %d/%d messages\n", consumed, MAX_MSGS); + TEST_ASSERT(consumed == MAX_MSGS, "Expected %d consumed, got %d", + MAX_MSGS, consumed); + + /* Records are either committed by the last commit_async or + * released on the broker side when the connection is closed. + * No lock wait needed. + * TODO KIP-932: When share consumer close is fully implemented, + * these tests may need to wait for the acquisition lock + * timeout before closing as close will commit acknowledged + * records in explicit acknowledgement mode. */ + + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Second consumer: should only get the 5 verification records. + * REJECT'd records are archived and not redelivered. */ + rkshare = test_create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer 2 consume_batch returned: rcvd=%zu, error=%s\n", + rcvd, error ? rd_kafka_error_string(error) : "none"); + if (error) { + rd_kafka_error_destroy(error); + } + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer 2 got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + received++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer 2 got %d messages (expected 5)\n", received); + TEST_ASSERT(received == 5, "Expected 5 verification records, got %d", + received); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Per-record commit_async — acknowledges each record individually + * and calls commit_async after each acknowledgement. + * + * Waits for acquisition lock timeout, then consumer 2 verifies all + * records were committed (0 redelivered). + * =================================================================== */ +static void do_test_per_record_commit_async(void) { + const char *topic; + const char *group = "commit-async-per-record"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0, received = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-per-rec", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, MAX_MSGS); + + rkshare = test_create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume all records, ACCEPT each individually with + * commit_async after every record */ + while (consumed < MAX_MSGS && attempts++ < 100) { + size_t rcvd_inner; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd_inner = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd_inner; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT( + !error, "commit_async at msg %d failed: %s", + consumed, + error ? rd_kafka_error_string(error) : ""); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumed %d/%d messages with per-record commit_async\n", + consumed, MAX_MSGS); + TEST_ASSERT(consumed == MAX_MSGS, "Expected %d, got %d", MAX_MSGS, + consumed); + + /* Records are either committed by the last commit_async or + * released on the broker side when the connection is closed. + * No lock wait needed. + * TODO KIP-932: When share consumer close is fully implemented, + * these tests may need to wait for the acquisition lock + * timeout before closing as close will commit acknowledged + * records in explicit acknowledgement mode. */ + + /* Wait for async commits to propagate */ + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Second consumer: should only get the 5 verification records. + * All previous records were ACCEPT'd via per-record commit_async. */ + rkshare = test_create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer 2 consume_batch returned: rcvd=%zu, error=%s\n", + rcvd, error ? rd_kafka_error_string(error) : "none"); + if (error) { + rd_kafka_error_destroy(error); + } + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer 2 got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + received++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer 2 got %d messages (expected 5)\n", received); + TEST_ASSERT(received == 5, "Expected 5 verification records, got %d", + received); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Mock broker helpers. + * =================================================================== */ + +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(3, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + /* Set auto.offset.reset=earliest so tests that produce + * before consuming see all records. */ + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + return ctx; +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static rd_kafka_share_t *new_share_consumer(const char *bootstraps, + const char *group_id, + const char *ack_mode) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "share.acknowledgement.mode", ack_mode); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + return rkshare; +} + +static void count_share_requests(rd_kafka_mock_cluster_t *mcluster, + int *share_fetch_cntp, + int *share_ack_cntp) { + size_t cnt; + size_t i; + rd_kafka_mock_request_t **requests; + + *share_fetch_cntp = 0; + *share_ack_cntp = 0; + + requests = rd_kafka_mock_get_requests(mcluster, &cnt); + + for (i = 0; i < cnt; i++) { + int16_t api_key = rd_kafka_mock_request_api_key(requests[i]); + if (api_key == RD_KAFKAP_ShareFetch) + (*share_fetch_cntp)++; + else if (api_key == RD_KAFKAP_ShareAcknowledge) + (*share_ack_cntp)++; + } + + rd_kafka_mock_request_destroy_array(requests, cnt); +} + + +/* =================================================================== + * Mock broker — inflight request caching. + * + * Consumes records, acknowledges each individually and calls + * commit_async after each acknowledgement. Verifies ShareFetch + * request count < commit_async call count, proving acks are cached + * when a request is already inflight (rkb_share_fetch_enqueued=true). + * =================================================================== */ +static void do_test_mock_inflight_caching(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + const char *topic = "mock-inflight-cache"; + const char *t = topic; + const int msgcnt = 100; + int consumed = 0, i = 0; + int share_fetch_cnt, share_ack_cnt; + int commit_cnt = 0; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + test_produce_msgs_simple(ctx.producer, topic, RD_KAFKA_PARTITION_UA, + msgcnt); + + rkshare = + new_share_consumer(ctx.bootstraps, "sg-mock-inflight", "explicit"); + + subscribe_consumer(rkshare, &t, 1); + + /* Clear and start tracking requests before the consume+ack+commit + * loop */ + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + /* Consume records batch-by-batch, acknowledging each record + * individually and calling commit_async after each acknowledgement. + * Most commit_async calls will find an inflight request and + * cache the acks in rkb_share_async_ack_details instead of + * sending a new ShareFetch request. */ + while (consumed < msgcnt && i < 30) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + i++; + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT( + !error, "commit_async at msg %d failed: %s", + consumed, + error ? rd_kafka_error_string(error) : ""); + commit_cnt++; + } + } + rd_kafka_messages_destroy(batch); + } + TEST_SAY("Mock: consumed %d/%d\n", consumed, msgcnt); + TEST_ASSERT(consumed == msgcnt, "Expected %d, got %d", msgcnt, + consumed); + + /* Wait for async ops to complete before counting requests */ + rd_sleep(3); + + count_share_requests(ctx.mcluster, &share_fetch_cnt, &share_ack_cnt); + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + /* We called commit_async 100 times but the total number of + * ShareFetch + ShareAcknowledge requests should be much fewer + * due to inflight caching. Ack-only requests use ShareAcknowledge + * RPC instead of ShareFetch. */ + TEST_SAY( + "Mock: commit_async calls=%d, ShareFetch requests=%d, " + "ShareAcknowledge requests=%d\n", + commit_cnt, share_fetch_cnt, share_ack_cnt); + TEST_ASSERT(share_ack_cnt > 0, + "Expected at least one ShareAcknowledge request, got %d", + share_ack_cnt); + TEST_ASSERT(share_fetch_cnt + share_ack_cnt < commit_cnt, + "Expected fewer total share requests (%d + %d = %d) than " + "commit_async calls (%d) due to inflight caching", + share_fetch_cnt, share_ack_cnt, + share_fetch_cnt + share_ack_cnt, commit_cnt); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Acquisition lock timeout — verify that records are redelivered + * to the SAME consumer after lock expiry when not acknowledged. + * + * TODO KIP-932: Move this to 0171 file maybe when admin client + * related things are finalized in that test case. + * + * Steps: + * 1. Produce 10 records. + * 2. Consume all 10 — verify delivery_count == 1. + * 3. Do NOT acknowledge or commit. + * 4. Wait for the 3 s acquisition lock to expire. + * 5. Consume again — verify the same 10 records arrive with + * delivery_count == 2. + * =================================================================== */ +static void do_test_lock_timeout_redelivery(void) { + const char *topic; + const char *group = "commit-async-lock-timeout"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed1 = 0, consumed2 = 0; + int attempts; + const int msg_cnt = 10; + + SUB_TEST("Lock timeout redelivery to same consumer"); + + topic = test_mk_topic_name("0173-ca-lock-to", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, msg_cnt); + + rkshare = test_create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + /* First consume: get all records, verify delivery_count == 1. + * Do NOT call commit_async — records stay ACQUIRED. */ + attempts = 0; + while (consumed1 == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT( + rd_kafka_message_delivery_count(rkm) == 1, + "First consume: expected delivery_count=1, " + "got %d at offset %" PRId64, + rd_kafka_message_delivery_count(rkm), + rkm->offset); + consumed1++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("First consume: got %d/%d records (not acknowledged)\n", + consumed1, msg_cnt); + TEST_ASSERT(consumed1 > 0, "Expected records on first consume, got 0"); + + /* Wait for acquisition lock to expire (3 s + buffer) */ + TEST_SAY("Waiting 4 s for acquisition lock to expire...\n"); + rd_sleep(4); + + /* Second consume on the SAME consumer: records should be + * redelivered with delivery_count == 2. */ + attempts = 0; + while (consumed2 == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT( + rd_kafka_message_delivery_count(rkm) == 2, + "Second consume: expected " + "delivery_count=2, got %d at " + "offset %" PRId64, + rd_kafka_message_delivery_count(rkm), + rkm->offset); + consumed2++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Second consume: got %d/%d redelivered records\n", consumed2, + msg_cnt); + TEST_ASSERT(consumed2 > 0, "Expected redelivered records, got 0"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test: commit_async callback invocation. + * + * Verifies that share_acknowledgement_commit_cb is invoked after + * commit_async when acks are piggybacked on ShareFetch. + * =================================================================== */ +static void do_test_commit_async_callback(void) { + const char *topic; + const char *group = "commit-async-callback"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + test_ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-callback", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 50); + + rkshare = + test_create_share_consumer_with_cb(group, "implicit", &state, NULL); + const char *grp_conf[] = {"share.auto.offset.reset", "SET", "earliest"}; + test_alter_group_configurations(group, grp_conf, 1); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume some messages */ + while (consumed < 20 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumed %d messages\n", consumed); + TEST_ASSERT(consumed > 0, "Expected to consume some messages"); + + /* Call commit_async to trigger callback */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Wait for callback */ + test_wait_for_cb_with_poll(&state, rkshare, 1, 10000); + + TEST_SAY("Callback count=%d, total_offsets=%zu, last_err=%s\n", + state.callback_cnt, state.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + + TEST_ASSERT(state.callback_cnt >= 1, + "Expected at least 1 callback, got %d", state.callback_cnt); + TEST_ASSERT(state.total_offsets > 0, + "Expected offsets in callback, got %zu", + state.total_offsets); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Partial-batch commit_async semantics. + * + * In explicit mode, the next consume_batch must error while any record + * from the previous batch is un-acked. After the remaining records are + * acked and a second commit_async is issued, consume_batch can drive + * the piggybacked acks and the commit callback fires for all offsets. + * =================================================================== */ +static void do_test_partial_batch_commit_async(void) { + const char *topic; + const char *group = "commit-async-partial-batch"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_resp_err_t ack_err; + rd_kafka_messages_t *batches[CONSUME_ARRAY] = {NULL}; + size_t batch_cnt = 0; + size_t total = 0; + size_t j; + test_ack_cb_state_t state = {0}; + const int msg_cnt = 5; + int attempts = 0; + size_t to_ack_first; + size_t acked = 0; + size_t seen = 0; + int wait_elapsed = 0; + rd_kafka_messages_t *wait_batch = NULL; + rd_kafka_error_t *wait_err = NULL; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-partial", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, msg_cnt); + + rkshare = + test_create_share_consumer_with_cb(group, "explicit", &state, NULL); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume all msg_cnt records (may take multiple polls). Keep each + * returned batch alive so the message pointers remain valid for + * later acknowledgement. */ + while (total < (size_t)msg_cnt && attempts++ < 30) { + rd_kafka_messages_t *poll_batch = NULL; + + error = rd_kafka_share_poll(rkshare, 3000, &poll_batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + batches[batch_cnt++] = poll_batch; + total += rd_kafka_messages_count(poll_batch); + } + TEST_ASSERT(total == (size_t)msg_cnt, + "Expected exactly %d records, got %zu", msg_cnt, total); + + /* Acknowledge first 3 records across the accumulated batches; leave + * the remainder un-acked. */ + to_ack_first = 3; + acked = 0; + for (j = 0; j < batch_cnt && acked < to_ack_first; j++) { + size_t k, n = rd_kafka_messages_count(batches[j]); + for (k = 0; k < n && acked < to_ack_first; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batches[j], k); + ack_err = rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "First-half ACCEPT %zu failed: %s", acked, + rd_kafka_err2str(ack_err)); + acked++; + } + } + + /* commit_async itself succeeds — it just schedules. */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "First commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* While un-acked records remain in the batch, share_poll must + * return _STATE rather than proceed. */ + rd_kafka_messages_t *dummy = NULL; + rd_kafka_error_t *e = rd_kafka_share_poll(rkshare, 1000, &dummy); + TEST_ASSERT(e != NULL, + "Expected share_poll to return _STATE while " + "un-acked records remain, got NULL error"); + TEST_ASSERT(rd_kafka_error_code(e) == RD_KAFKA_RESP_ERR__STATE, + "Expected _STATE, got %s", + rd_kafka_err2name(rd_kafka_error_code(e))); + TEST_SAY("Got expected _STATE from share_poll: %s\n", + rd_kafka_error_string(e)); + rd_kafka_error_destroy(e); + rd_kafka_messages_destroy(dummy); + + /* Acknowledge the remaining records across the batches. */ + for (j = 0; j < batch_cnt; j++) { + size_t k, n = rd_kafka_messages_count(batches[j]); + for (k = 0; k < n; k++) { + rd_kafka_message_t *rkm; + if (seen < to_ack_first) { + seen++; + continue; + } + rkm = rd_kafka_messages_get(batches[j], k); + ack_err = rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Second-half ACCEPT %zu (offset=%" PRId64 + ") failed: %s", + seen, rkm->offset, + rd_kafka_err2str(ack_err)); + seen++; + } + } + + /* Second commit_async commits the remaining acks */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "Second commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Now share_poll can proceed and drive piggybacked acks. The two + * commits may surface as one or two callbacks depending on broker + * coalescing, so wait on total_offsets rather than callback_cnt: + * the first commit's callback can fire during the _STATE-returning + * poll above, which would short-circuit a callback_cnt>=1 wait + * before the second commit lands. */ + while (state.total_offsets < (size_t)total && wait_elapsed < 15000) { + wait_batch = NULL; + wait_err = rd_kafka_share_poll(rkshare, 100, &wait_batch); + if (wait_err) + rd_kafka_error_destroy(wait_err); + rd_kafka_messages_destroy(wait_batch); + wait_elapsed += 100; + } + TEST_ASSERT(state.callback_cnt >= 1, + "Expected ack commit callback to fire after full batch " + "ack + commit_async, got %d", + state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&state) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Callback errored: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + TEST_ASSERT(state.total_offsets >= (size_t)total, + "Expected callback to report %zu offsets, got %zu", total, + state.total_offsets); + TEST_SAY("Partial commit callbacks=%d, total_offsets=%zu\n", + state.callback_cnt, state.total_offsets); + + for (j = 0; j < batch_cnt; j++) + rd_kafka_messages_destroy(batches[j]); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Implicit ack callback fires from next poll's piggybacked acks + * without any explicit commit_async call. + * =================================================================== */ +static void do_test_implicit_callback_no_explicit_commit(void) { + const char *topic; + const char *group = "commit-async-implicit-no-commit"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd, j; + test_ack_cb_state_t state = {0}; + const int msg_cnt = 20; + int consumed = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-impl-no-commit", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, msg_cnt); + + rkshare = + test_create_share_consumer_with_cb(group, "implicit", &state, NULL); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* First poll: drain all produced records. share_poll typically + * returns the whole broker-side RecordBatch (so all msg_cnt records + * arrive in one call), but the loop tolerates split deliveries. */ + while (consumed < msg_cnt && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed == msg_cnt, + "Expected to consume %d messages, got %d", msg_cnt, + consumed); + + /* Subsequent polls without any commit_async — callback should still + * fire via piggybacked acks on the next ShareFetch. */ + attempts = 10; + while (attempts-- > 0 && state.callback_cnt == 0) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(state.callback_cnt >= 1, + "Expected callback from implicit piggybacked acks (no " + "explicit commit), got %d", + state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&state) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Callback errored: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Lock-expiry surfaces an error in the ack commit callback. + * + * Explicit mode + short lock duration: poll a record, acknowledge it, + * sleep past the lock expiry, then commit_async. The broker should + * reject the stale ack and the error must be reported via the + * acknowledgement commit callback. + * =================================================================== */ +static void do_test_lock_expiry_callback_err(void) { + const char *topic; + const char *group = "commit-async-lock-expiry-cb"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_resp_err_t ack_err; + rd_kafka_messages_t *batch = NULL; + size_t j; + size_t consumed = 0; + int attempts = 0; + test_ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-lock-cb", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 3); + + rkshare = + test_create_share_consumer_with_cb(group, "explicit", &state, NULL); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume records (acquires lock) */ + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + consumed = rd_kafka_messages_count(batch); + } + TEST_ASSERT(consumed > 0, "Expected to consume records, got 0"); + + /* Acknowledge all consumed records explicitly */ + for (j = 0; j < consumed; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + ack_err = rd_kafka_share_acknowledge_type( + rkshare, rkm, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "ACCEPT failed: %s", rd_kafka_err2str(ack_err)); + } + + /* Sleep past acquisition-lock expiry (3s + buffer) */ + TEST_SAY("Sleeping 4 s for lock expiry before commit_async...\n"); + rd_sleep(4); + + /* commit_async on stale acks - broker should reject them */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async returned error directly: %s", + error ? rd_kafka_error_string(error) : ""); + + test_wait_for_cb_with_poll(&state, rkshare, 1, 15000); + + TEST_SAY("Callback count=%d, total_offsets=%zu, last_err=%s\n", + state.callback_cnt, state.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + + TEST_ASSERT(state.callback_cnt >= 1, + "Expected callback to fire for stale ack, got %d", + state.callback_cnt); + /* Broker rejects acks against records whose acquisition lock has + * expired with INVALID_RECORD_STATE. */ + TEST_ASSERT(test_ack_cb_state_first_err(&state) == + RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE, + "Expected INVALID_RECORD_STATE in callback after lock " + "expiry, got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state))); + + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + SUB_TEST_PASS(); +} + + +typedef struct { + test_ack_cb_state_t base; + int saw_consumer_name; +} k44_state_t; + +static void k44_safe_api_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + k44_state_t *st = opaque; + rd_kafka_t *rk; + + /* Reuse base counters via the shared helper */ + test_share_ack_cb(rkshare, partitions, err, &st->base); + + /* Safely read consumer identity from within the callback */ + rk = test_share_consumer_get_rk(rkshare); + if (rk && rd_kafka_name(rk)) + st->saw_consumer_name = 1; +} + +/* =================================================================== + * Callback safely reads consumer state and the consumer keeps working + * afterwards. + * + * Implicit mode: poll a batch, commit_async to drive piggyback acks, + * wait for the ack callback. The callback calls + * test_share_consumer_get_rk()+rd_kafka_name() to prove that + * consumer-state APIs are safe to invoke from within the callback. + * After the callback runs, a second round of consume_batch must still + * return records, proving the consumer is not wedged. + * =================================================================== */ +static void do_test_safe_api_from_callback(void) { + const char *topic; + const char *group = "commit-async-safe-api-cb"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd, j; + int consumed = 0; + int second_round = 0; + int attempts = 0; + k44_state_t st = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-safe-api-cb", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 10); + + rkshare = test_create_share_consumer_with_cb( + group, "implicit", (test_ack_cb_state_t *)&st, k44_safe_api_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* First-round consumption */ + while (consumed < 5 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Drive piggyback acks - this fires the callback */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed"); + test_wait_for_cb_with_poll(&st.base, rkshare, 1, 10000); + + TEST_ASSERT(st.base.callback_cnt >= 1, "Expected callback, got %d", + st.base.callback_cnt); + TEST_ASSERT(st.saw_consumer_name, + "Callback should have read consumer name via " + "test_share_consumer_get_rk + rd_kafka_name"); + + /* After callback has run, the consumer must still function. */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + attempts = 0; + while (second_round < 5 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + second_round++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(second_round > 0, + "Consumer stopped working after callback invocation"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&st.base); + + SUB_TEST_PASS(); +} + + +typedef struct { + test_ack_cb_state_t base; + int side_effect_runs; +} k45_state_t; + +static void +k45_side_effect_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + k45_state_t *st = opaque; + + test_share_ack_cb(rkshare, partitions, err, &st->base); + + /* Heavy-ish side effect: simulate user application work. */ + st->side_effect_runs++; + rd_sleep(0); /* yield - safe no-op time pass */ +} + +/* =================================================================== + * Application-side side effects in the ack callback do not crash or + * hang the poll thread. + * + * Implicit mode: poll one batch, commit_async, and wait for the + * callback. The callback bumps a side-effect counter and yields + * via rd_sleep(0) to simulate user work. We assert the side effect + * ran and that the consumer is still healthy on a follow-up + * consume_batch / commit_async round. + * =================================================================== */ +static void do_test_callback_side_effects_dont_break(void) { + const char *topic; + const char *group = "commit-async-side-effect-cb"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd, j; + int consumed = 0; + int post_count = 0; + int attempts = 0; + k45_state_t st = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-ca-side-effect-cb", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 10); + + rkshare = test_create_share_consumer_with_cb( + group, "implicit", (test_ack_cb_state_t *)&st, k45_side_effect_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume the produced records */ + while (consumed < 10 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed > 0, "Expected to consume records, got 0"); + + /* Drive the side-effect callback at least once via commit_async + + * wait_for_cb_with_poll. */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + test_wait_for_cb_with_poll(&st.base, rkshare, 1, 10000); + + TEST_ASSERT(st.base.callback_cnt >= 1, + "Expected callback to fire at least once, got %d", + st.base.callback_cnt); + TEST_ASSERT(st.side_effect_runs >= 1, + "Side-effect did not run inside callback (%d runs vs " + "%d callbacks)", + st.side_effect_runs, st.base.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&st.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Callback err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&st.base))); + TEST_SAY("consumed=%d, callbacks=%d, side_effects=%d\n", consumed, + st.base.callback_cnt, st.side_effect_runs); + + /* Consumer must still be usable after the side-effect callback ran: + * produce more records and verify poll still works. */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + attempts = 0; + while (post_count == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + post_count++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(post_count > 0, + "Consumer stopped working after side-effect callback"); + TEST_SAY("Post-callback poll returned %d msgs\n", post_count); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&st.base); + + SUB_TEST_PASS(); +} + +static void do_test_change_callback(void) { + const char *topic; + const char *group = "change-callback"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd, j; + int consumed = 0; + int attempts = 0; + int cb_a_after = 0; + test_ack_cb_state_t state_a = {0}; + test_ack_cb_state_t state_b = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-change-callback", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Phase 1: produce first batch of messages */ + test_produce_msgs_simple(common_producer, topic, 0, 20); + + /* Create consumer with callback A registered */ + rkshare = test_create_share_consumer_with_cb(group, "explicit", + &state_a, NULL); + /* Set offset reset to earliest */ + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Phase 1: consume the first batch and commit with callback A */ + while (consumed < 20 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + consumed++; + rd_kafka_share_acknowledge(rkshare, rkm); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(consumed > 0, "Expected to consume some messages"); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "First commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Wait for callback A */ + test_wait_for_cb_with_poll(&state_a, rkshare, 1, 10000); + TEST_ASSERT(state_a.callback_cnt == 1, + "Expected callback A to be invoked once, got %d", + state_a.callback_cnt); + TEST_ASSERT(state_b.callback_cnt == 0, + "Expected callback B NOT to be invoked yet, got %d", + state_b.callback_cnt); + TEST_SAY("Phase 1: callback A invoked %d times, total offsets=%zu\n", + state_a.callback_cnt, state_a.total_offsets); + + /* Phase 2: replace callback A with callback B (different opaque) */ + error = rd_kafka_share_set_acknowledgement_commit_cb( + rkshare, test_share_ack_cb, &state_b); + TEST_ASSERT(error == NULL, + "Expected to change callback successfully, got %s", + rd_kafka_error_string(error)); + + /* Remember current callback A count so we can verify it doesn't + * increase after the swap. */ + cb_a_after = state_a.callback_cnt; + + /* Produce a fresh batch so phase 2 has new messages to consume */ + test_produce_msgs_simple(common_producer, topic, 0, 20); + + /* Consume the fresh batch */ + consumed = 0; + attempts = 0; + while (consumed < 20 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + consumed++; + rd_kafka_share_acknowledge(rkshare, rkm); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(consumed > 0, + "Expected to consume some messages after callback swap"); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "Second commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Wait for callback B to be invoked */ + test_wait_for_cb_with_poll(&state_b, rkshare, 1, 10000); + TEST_ASSERT(state_b.callback_cnt == 1, + "Expected callback B to be invoked once, got %d", + state_b.callback_cnt); + + /* Critical assertion: callback A's count must NOT have increased + * since the swap - the new callback should be receiving the new + * results, not the old one. */ + TEST_ASSERT(state_a.callback_cnt == cb_a_after, + "Callback A was invoked AFTER swap (was %d, now %d) " + "- new callback should have received the results", + cb_a_after, state_a.callback_cnt); + + TEST_SAY( + "Phase 2: callback A unchanged at %d, callback B invoked " + "%d times, total offsets=%zu\n", + state_a.callback_cnt, state_b.callback_cnt, state_b.total_offsets); + + /* Phase 3: unregister callback (NULL). After this, no callback + * should be invoked even when new acks are committed. */ + int cb_a_before_phase3 = state_a.callback_cnt; + int cb_b_before_phase3 = state_b.callback_cnt; + + error = + rd_kafka_share_set_acknowledgement_commit_cb(rkshare, NULL, NULL); + TEST_ASSERT(error == NULL, "Failed to unregister callback: %s", + rd_kafka_error_string(error)); + + /* Produce a fresh batch to trigger more acknowledgements */ + test_produce_msgs_simple(common_producer, topic, 0, 20); + + /* Consume the fresh batch */ + consumed = 0; + attempts = 0; + while (consumed < 20 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + consumed++; + rd_kafka_share_acknowledge(rkshare, rkm); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed > 0, + "Expected to consume some messages after unregister"); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "Third commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Poll for a while to give any pending callbacks a chance to fire. + * We can't use test_wait_for_cb_with_poll because we EXPECT no + * callback - so we poll manually with a fixed budget. */ + int poll_iters = 20; + while (poll_iters-- > 0) { + error = rd_kafka_share_poll(rkshare, 200, &batch); + if (error) + rd_kafka_error_destroy(error); + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + consumed++; + rd_kafka_share_acknowledge(rkshare, rkm); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Critical assertion: neither callback should have been invoked + * after the unregister. */ + TEST_ASSERT(state_a.callback_cnt == cb_a_before_phase3, + "Callback A was invoked after unregister (was %d, now %d)", + cb_a_before_phase3, state_a.callback_cnt); + TEST_ASSERT(state_b.callback_cnt == cb_b_before_phase3, + "Callback B was invoked after unregister (was %d, now %d)", + cb_b_before_phase3, state_b.callback_cnt); + + TEST_SAY( + "Phase 3: no callbacks invoked after unregister " + "(A=%d, B=%d unchanged)\n", + state_a.callback_cnt, state_b.callback_cnt); + + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); + test_ack_cb_state_destroy(&state_a); + test_ack_cb_state_destroy(&state_b); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test: reentrancy protection - share consumer APIs cannot be called + * from within the acknowledgement callback. + * + * Verifies that calling any share consumer API from inside the + * registered acknowledgement callback returns RD_KAFKA_RESP_ERR__STATE + * (for rd_kafka_resp_err_t APIs) or a STATE-coded error object + * (for rd_kafka_error_t APIs). + * =================================================================== */ +typedef struct reentrancy_check_state_s { + test_ack_cb_state_t base; /**< Must be first - we cast to base */ + int rejections; /**< Count of correctly rejected calls */ + int failures; /**< Count of calls that did NOT reject */ + rd_kafka_share_t *rkshare; +} reentrancy_check_state_t; + +static void +reentrancy_check_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + reentrancy_check_state_t *st = (reentrancy_check_state_t *)opaque; + rd_kafka_error_t *error_obj; + rd_kafka_resp_err_t resp_err; + rd_kafka_messages_t *batch = NULL; + rd_kafka_topic_partition_list_t *subs; + + /* Update base state for callback tracking */ + test_ack_cb_state_push_err(&st->base, err); + if (partitions) { + const rd_kafka_share_partition_offsets_t *p = + rd_kafka_share_partition_offsets_list_get(partitions, 0); + if (p) + st->base.total_offsets += + rd_kafka_share_partition_offsets_offsets_cnt(p); + } + + /* Try rd_kafka_share_poll - should fail with _STATE */ + error_obj = rd_kafka_share_poll(rkshare, 100, &batch); + if (error_obj && + rd_kafka_error_code(error_obj) == RD_KAFKA_RESP_ERR__STATE) { + st->rejections++; + rd_kafka_error_destroy(error_obj); + } else { + st->failures++; + if (error_obj) + rd_kafka_error_destroy(error_obj); + } + rd_kafka_messages_destroy(batch); + + /* Try rd_kafka_share_commit_async - should fail with _STATE */ + error_obj = rd_kafka_share_commit_async(rkshare); + if (error_obj && + rd_kafka_error_code(error_obj) == RD_KAFKA_RESP_ERR__STATE) { + st->rejections++; + rd_kafka_error_destroy(error_obj); + } else { + st->failures++; + if (error_obj) + rd_kafka_error_destroy(error_obj); + } + + /* Try rd_kafka_share_consumer_close - should fail with _STATE */ + error_obj = rd_kafka_share_consumer_close(rkshare); + if (error_obj && + rd_kafka_error_code(error_obj) == RD_KAFKA_RESP_ERR__STATE) { + st->rejections++; + rd_kafka_error_destroy(error_obj); + } else { + st->failures++; + if (error_obj) + rd_kafka_error_destroy(error_obj); + } + + /* Try rd_kafka_share_subscribe - should fail with _STATE */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, "dummy", RD_KAFKA_PARTITION_UA); + resp_err = rd_kafka_share_subscribe(rkshare, subs); + if (resp_err == RD_KAFKA_RESP_ERR__STATE) + st->rejections++; + else + st->failures++; + rd_kafka_topic_partition_list_destroy(subs); + + /* Try rd_kafka_share_unsubscribe - should fail with _STATE */ + resp_err = rd_kafka_share_unsubscribe(rkshare); + if (resp_err == RD_KAFKA_RESP_ERR__STATE) + st->rejections++; + else + st->failures++; + + /* Try rd_kafka_share_set_acknowledgement_commit_cb - should fail with + * _STATE because you can't change the callback from within itself */ + error_obj = + rd_kafka_share_set_acknowledgement_commit_cb(rkshare, NULL, NULL); + if (error_obj && + rd_kafka_error_code(error_obj) == RD_KAFKA_RESP_ERR__STATE) + st->rejections++; + else + st->failures++; + RD_IF_FREE(error_obj, rd_kafka_error_destroy); +} + +static void do_test_reentrancy_protection(void) { + const char *topic; + const char *group = "reentrancy-protection"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + size_t rcvd, j; + int consumed = 0; + int attempts = 0; + reentrancy_check_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0173-reentrancy", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 50); + + /* Create consumer without callback - we'll set it at runtime */ + rkshare = test_create_share_consumer(group, "explicit"); + state.rkshare = rkshare; + + /* Register the reentrancy-checking callback at runtime */ + error = rd_kafka_share_set_acknowledgement_commit_cb( + rkshare, reentrancy_check_cb, &state); + TEST_ASSERT(error == NULL, "Failed to set callback: %s", + rd_kafka_error_string(error)); + + /* Set offset reset to earliest */ + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume messages to trigger acknowledgement and callback invocation + */ + while (consumed < 50 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + consumed++; + rd_kafka_share_acknowledge(rkshare, rkm); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed > 0, "Expected to consume some messages"); + + /* commit_async triggers the callback */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async failed: %s", + error ? rd_kafka_error_string(error) : ""); + + /* Wait for callback to be invoked at least once */ + test_wait_for_cb_with_poll(&state.base, rkshare, 1, 10000); + TEST_ASSERT(state.base.callback_cnt == 1, + "Expected callback to be invoked, got %d", + state.base.callback_cnt); + + TEST_SAY("Callback invoked %d times, rejections=%d, failures=%d\n", + state.base.callback_cnt, state.rejections, state.failures); + + /* Every callback invocation tries 6 share consumer APIs, all of + * which must reject the call from within the callback. */ + TEST_ASSERT(state.failures == 0, + "Expected 0 failures (all APIs should reject), got %d", + state.failures); + TEST_ASSERT(state.rejections == 6, "Expected 6 rejections, got %d", + state.rejections); + + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + SUB_TEST_PASS(); +} + + + +int main_0173_share_consumer_commit_async(int argc, char **argv) { + test_timeout_set(120); + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + do_test_implicit_second_consumer(); + do_test_explicit_second_consumer(); + do_test_mixed_acks_second_consumer(); + do_test_multi_topic_partition(); + do_test_produce_consume_loop(); + do_test_multi_round_mixed_second_consumer(); + do_test_no_pending_acks(); + do_test_multiple_commit_async_calls(); + do_test_commit_between_produces(); + do_test_all_release_second_consumer(); + do_test_all_reject_second_consumer(); + do_test_per_record_commit_async(); + do_test_lock_timeout_redelivery(); + /* Callback test */ + do_test_commit_async_callback(); + + /* Partial-batch / callback edge cases */ + do_test_partial_batch_commit_async(); + do_test_implicit_callback_no_explicit_commit(); + do_test_lock_expiry_callback_err(); + do_test_safe_api_from_callback(); + do_test_callback_side_effects_dont_break(); + + /* Callback management and reentrancy */ + do_test_change_callback(); + do_test_reentrancy_protection(); + + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} + +int main_0173_share_consumer_commit_async_local(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + do_test_mock_inflight_caching(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0174-share_consumer_concurrency.c b/lib/librdkafka-2.15.0/tests/0174-share_consumer_concurrency.c new file mode 100644 index 00000000000..c14605dd009 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0174-share_consumer_concurrency.c @@ -0,0 +1,1434 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "rdkafka.h" + +/** + * @name Share Consumer Concurrency and Stress Tests + * + * Tests share consumer behavior under concurrent producer/consumer scenarios. + * Each test uses a config-based structure. Producers and consumers each run + * in their own dedicated threads — there is no main-thread round-robin + * polling. The main test thread orchestrates start/stop and verifies + * end-state counters. The per-record invariant validated is first-delivery + * count (each produced record is first-delivered exactly once; redeliveries + * are counted separately). + */ + +#define MAX_TOPICS 16 +#define MAX_PARTITIONS 16 +#define MAX_CONSUMERS 16 +#define MAX_PRODUCERS 10 +#define MAX_CONSUMER_POOL 32 +#define BATCH_SIZE 10000 + +/* Save test_curr for threads (test_curr is TLS) */ +static struct test *this_test; + +/*************************************************************************** + * Test Configuration and State + ***************************************************************************/ + +/** + * @brief Configuration for a concurrent share consumer test + */ +typedef struct { + int consumer_cnt; /**< Number of consumers to create */ + int producer_cnt; /**< Number of producer threads */ + int topic_cnt; /**< Number of topics */ + int partitions[MAX_TOPICS]; /**< Partitions per topic */ + int msgs_per_partition; /**< Messages to produce per partition */ + const char *group_name; /**< Share group name */ + const char *test_name; /**< Test description */ + int max_attempts; /**< Max poll attempts (0 = default) */ + int consumer_delay_ms; /**< Delay between consumer polls */ + int producer_delay_ms; /**< Delay between produces */ + rd_bool_t explicit_ack; /**< Use explicit acknowledgement */ + rd_bool_t staggered_start; /**< Start producers/consumers at + different times */ +} concurrent_test_config_t; + +/** + * @brief Shared state for concurrent test threads + */ +typedef struct { + mtx_t lock; /**< Mutex for state access */ + int total_produced; /**< Total messages produced */ + int total_consumed; /**< Total messages consumed + * (includes redeliveries) */ + int total_first_delivery; /**< Messages with delivery_count == 1 */ + int total_duplicates; /**< Messages with delivery_count > 1 */ + int expected_total; /**< Expected total messages */ + rd_bool_t producers_done; /**< All producers finished */ + int producers_remaining; /**< Producer threads still running */ + rd_bool_t test_failed; /**< Test failure flag */ + char topics[MAX_TOPICS][128]; /**< Topic names */ + int topic_cnt; /**< Number of topics */ + const char *group_name; /**< Share group name */ + rd_bool_t explicit_ack; /**< Use explicit acknowledgement */ +} concurrent_test_state_t; + +/** + * @brief Arguments for producer thread + */ +typedef struct { + concurrent_test_state_t *state; + int producer_id; + int msgs_to_produce; + int delay_ms; + int partitions[MAX_TOPICS]; +} producer_thread_args_t; + +/** + * @brief Per-consumer-thread context. + * + * One share consumer per thread; main thread sets @c run to rd_false + * (under state->lock) to ask the consumer to wind down. The consumer + * thread sets @c done = rd_true under the same lock right before it + * returns from consumer_thread_func, so the main thread can poll for a + * clean exit instead of blocking forever in thrd_join when a consumer + * is wedged inside librdkafka. + */ +typedef struct { + concurrent_test_state_t *state; + int consumer_id; + rd_bool_t run; + rd_bool_t done; /**< Set true just before consumer thread exits */ + int poll_delay_ms; /**< Sleep between polls (from config) */ +} consumer_thread_args_t; + +/*************************************************************************** + * Thread Functions + ***************************************************************************/ + +/** + * @brief Producer thread function + * + * Produces messages to all topics/partitions in round-robin fashion. + */ +static int producer_thread_func(void *arg) { + producer_thread_args_t *args = (producer_thread_args_t *)arg; + concurrent_test_state_t *state = args->state; + rd_kafka_t *producer; + rd_kafka_conf_t *conf; + int produced = 0; + int t, p; + rd_kafka_resp_err_t err; + char value[64]; + + /* Restore the TLS test_curr pointer for this thread. The + * exp_dr_status / ignore_dr_err fields on the struct itself are + * set once by the main thread before any worker spawns + * (see run_concurrent_test / do_test_chaos_consumer_lifecycle) + * — touching them here would race against every other producer + * thread that aliases the same global tests[] entry. */ + test_curr = this_test; + + /* Enable idempotent producer so retries do not result in duplicate + * broker-side writes — without this, transient broker/network jitter + * under the heavy concurrent load these tests generate (e.g. + * test_max_concurrent with 10 producers × 5000 msgs) causes the + * broker to store extra records that the share consumer then + * receives as new offsets with delivery_count=1, inflating the + * consumed count above expected_total. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "enable.idempotence", "true"); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + producer = test_create_handle(RD_KAFKA_PRODUCER, conf); + + TEST_SAY("Producer %d: starting, will produce %d messages\n", + args->producer_id, args->msgs_to_produce); + + while (produced < args->msgs_to_produce) { + for (t = 0; + t < state->topic_cnt && produced < args->msgs_to_produce; + t++) { + for (p = 0; p < args->partitions[t] && + produced < args->msgs_to_produce; + p++) { + snprintf(value, sizeof(value), "prod%d-msg%d", + args->producer_id, produced); + + err = rd_kafka_producev( + producer, + RD_KAFKA_V_TOPIC(state->topics[t]), + RD_KAFKA_V_PARTITION(p), + RD_KAFKA_V_VALUE(value, strlen(value)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END); + + if (err) { + TEST_SAY( + "Producer %d: produce failed: %s\n", + args->producer_id, + rd_kafka_err2str(err)); + mtx_lock(&state->lock); + state->test_failed = rd_true; + mtx_unlock(&state->lock); + goto done; + } + + produced++; + + if (args->delay_ms > 0 && produced % 100 == 0) + rd_usleep(args->delay_ms * 1000, NULL); + } + } + + rd_kafka_flush(producer, 1000); + } + + rd_kafka_flush(producer, 10000); + + TEST_SAY("Producer %d: finished, produced %d messages\n", + args->producer_id, produced); + +done: + mtx_lock(&state->lock); + state->total_produced += produced; + if (--state->producers_remaining == 0) + state->producers_done = rd_true; + mtx_unlock(&state->lock); + + rd_kafka_destroy(producer); + free(args); + + return thrd_success; +} + +/** + * @brief Per-thread share consumer poll loop. + * + * One share consumer per thread. Creates the consumer, subscribes to all + * configured topics, then polls until @c args->run is cleared by the main + * thread. Records first-time deliveries vs redeliveries via + * rd_kafka_message_delivery_count() into shared counters. + */ +static int consumer_thread_func(void *arg) { + consumer_thread_args_t *args = (consumer_thread_args_t *)arg; + concurrent_test_state_t *state = args->state; + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + int t; + size_t m; + size_t rcvd; + rd_bool_t keep_running; + + test_curr = this_test; + + consumer = test_create_share_consumer( + state->group_name, state->explicit_ack ? "explicit" : NULL); + + subs = rd_kafka_topic_partition_list_new(state->topic_cnt); + for (t = 0; t < state->topic_cnt; t++) { + rd_kafka_topic_partition_list_add(subs, state->topics[t], + RD_KAFKA_PARTITION_UA); + } + rd_kafka_share_subscribe(consumer, subs); + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("Consumer thread %d: subscribed\n", args->consumer_id); + + while (1) { + int acked_in_batch = 0; + + mtx_lock(&state->lock); + keep_running = args->run; + mtx_unlock(&state->lock); + if (!keep_running) + break; + + err = rd_kafka_share_poll(consumer, 500, &batch); + if (err) { + TEST_SAY("Consumer thread %d: share_poll error: %s\n", + args->consumer_id, rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + rcvd = rd_kafka_messages_count(batch); + + /* Per-record atomic step: + * ACK → process (count into shared state under lock) → + * destroy. ack + process happen in the same iteration + * so there is no window where the broker can see an + * ACK for a record that this application has not also + * recorded as consumed in total_first_delivery / + * total_duplicates. The per-batch commit_sync below + * flushes the acks to the broker. */ + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + + if (!msg) + continue; + + if (msg->err) { + const char *msg_topic = + msg->rkt ? rd_kafka_topic_name(msg->rkt) + : "(no-topic)"; + TEST_SAY( + "Consumer thread %d: SKIPPING record " + "%s[%" PRId32 "] off=%" PRId64 + " because msg->err=%s\n", + args->consumer_id, msg_topic, + msg->partition, msg->offset, + rd_kafka_err2name(msg->err)); + continue; + } + + if (state->explicit_ack) { + rd_kafka_resp_err_t ack_err = + rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + if (ack_err != RD_KAFKA_RESP_ERR_NO_ERROR) { + TEST_SAY( + "Consumer thread %d: " + "acknowledge offset %" PRId64 + " failed: %s — skipping count " + "for this record\n", + args->consumer_id, msg->offset, + rd_kafka_err2name(ack_err)); + continue; + } + acked_in_batch++; + } + + /* Process the record at the same instant we + * acknowledged it. */ + { + int16_t dc = + rd_kafka_message_delivery_count(msg); + mtx_lock(&state->lock); + if (dc > 1) + state->total_duplicates++; + else + state->total_first_delivery++; + state->total_consumed++; + mtx_unlock(&state->lock); + } + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Flush the per-record acks to the broker before the + * next consume_batch or stop check. Per-partition + * results are inspected for errors. */ + if (state->explicit_ack && acked_in_batch > 0) { + rd_kafka_topic_partition_list_t *cs_result = NULL; + rd_kafka_error_t *commit_err = + rd_kafka_share_commit_sync(consumer, 5000, + &cs_result); + if (commit_err) { + TEST_SAY( + "Consumer thread %d: commit_sync failed " + "(acked_in_batch=%d): %s\n", + args->consumer_id, acked_in_batch, + rd_kafka_error_string(commit_err)); + rd_kafka_error_destroy(commit_err); + } else if (cs_result) { + int i; + for (i = 0; i < cs_result->cnt; i++) { + if (cs_result->elems[i].err != + RD_KAFKA_RESP_ERR_NO_ERROR) { + TEST_SAY( + "Consumer thread %d: " + "commit_sync per-part " + "err %s [%" PRId32 + "]: %s\n", + args->consumer_id, + cs_result->elems[i].topic, + cs_result->elems[i] + .partition, + rd_kafka_err2name( + cs_result->elems[i] + .err)); + } + } + } + if (cs_result) + rd_kafka_topic_partition_list_destroy( + cs_result); + } + + if (args->poll_delay_ms > 0) + rd_usleep(args->poll_delay_ms * 1000, NULL); + } + + TEST_SAY("Consumer thread %d: closing\n", args->consumer_id); + test_share_consumer_close(consumer); + test_share_destroy(consumer); + + /* Mark this thread as cleanly exited so the main thread's bounded + * wait can detect it without blocking in thrd_join. */ + mtx_lock(&state->lock); + args->done = rd_true; + mtx_unlock(&state->lock); + + return thrd_success; +} + +/*************************************************************************** + * Test Runner — fully threaded + * + * Every share consumer and every producer runs in its own dedicated + * thread. The main thread only orchestrates start/stop and verifies + * final counters. No round-robin main-thread polling. + ***************************************************************************/ + +/** + * @brief Run a concurrent share consumer test with every consumer in its + * own thread. + * + * @c config->max_attempts is interpreted as an upper bound on the total + * wait-for-completion time, measured in 500ms ticks (so max_attempts=100 + * gives a 50 s bound). When unset (0), the default is 240 ticks = 120 s. + */ +static void run_concurrent_test(const concurrent_test_config_t *config) { + concurrent_test_state_t state = {0}; + thrd_t producer_threads[MAX_PRODUCERS]; + thrd_t consumer_threads[MAX_CONSUMERS]; + consumer_thread_args_t consumer_args[MAX_CONSUMERS]; + rd_kafka_share_t *dummy_consumer; + int t, c, p; + int total_partitions = 0; + int msgs_per_producer; + int max_wait_ticks; + int wait_ticks; + int final_produced; + int final_consumed; + int final_first_delivery; + int final_duplicates; + rd_bool_t final_failed; + char unique_suffix[64]; + char unique_group[128]; + char unique_test_name[256]; + + TEST_ASSERT(config->consumer_cnt <= MAX_CONSUMERS, + "consumer_cnt %d exceeds MAX_CONSUMERS %d", + config->consumer_cnt, MAX_CONSUMERS); + + /* Generate a per-invocation unique suffix and append it to both + * the group name and the displayed test name. Avoids collisions + * across re-runs / parallel runs that share the same broker. */ + rd_snprintf(unique_suffix, sizeof(unique_suffix), "rnd%" PRIx64, + test_id_generate()); + rd_snprintf(unique_group, sizeof(unique_group), "%s-%s", + config->group_name, unique_suffix); + rd_snprintf(unique_test_name, sizeof(unique_test_name), "%s [%s]", + config->test_name, unique_suffix); + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== %s ===\n", unique_test_name); + TEST_SAY( + "============================================================\n"); + TEST_SAY( + "Threaded: %d consumer threads, %d producer threads, %d topics\n", + config->consumer_cnt, config->producer_cnt, config->topic_cnt); + + /* Save test context for threads (test_curr is TLS) */ + this_test = test_curr; + + /* Set the DR-callback expectations on the shared test struct + * ONCE on the main thread, before spawning any worker threads. + * test_curr is a TLS pointer but the struct it points at lives + * in the global tests[] array, so writing these fields from + * each producer thread (as it used to) is a benign-but-noisy + * data race that TSAN flags. */ + test_curr->exp_dr_status = (rd_kafka_msg_status_t)-1; + test_curr->ignore_dr_err = rd_true; + + /* Initialize state */ + mtx_init(&state.lock, mtx_plain); + state.topic_cnt = config->topic_cnt; + state.group_name = unique_group; + state.explicit_ack = config->explicit_ack; + state.producers_remaining = config->producer_cnt; + + for (t = 0; t < config->topic_cnt; t++) + total_partitions += config->partitions[t]; + state.expected_total = total_partitions * config->msgs_per_partition; + TEST_SAY( + "Total partitions: %d, msgs/partition: %d, expected total: %d\n", + total_partitions, config->msgs_per_partition, state.expected_total); + + /* Create topics */ + for (t = 0; t < config->topic_cnt; t++) { + rd_snprintf(state.topics[t], sizeof(state.topics[t]), "%s", + test_mk_topic_name("0174-concurrent", 1)); + test_create_topic_wait_exists(NULL, state.topics[t], + config->partitions[t], -1, + 60 * 1000); + TEST_SAY("Created topic %s with %d partitions\n", + state.topics[t], config->partitions[t]); + } + + /* Apply broker-side share group config (auto.offset.reset=earliest) + * via a throwaway consumer registration so the alter sticks before + * any worker subscribes. */ + dummy_consumer = test_create_share_consumer(unique_group, NULL); + test_share_set_auto_offset_reset(unique_group, "earliest"); + test_share_consumer_close(dummy_consumer); + test_share_destroy(dummy_consumer); + + /* Start producer threads */ + msgs_per_producer = state.expected_total / config->producer_cnt; + for (p = 0; p < config->producer_cnt; p++) { + producer_thread_args_t *args = rd_calloc(1, sizeof(*args)); + args->state = &state; + args->producer_id = p; + args->msgs_to_produce = msgs_per_producer; + if (p == config->producer_cnt - 1) + args->msgs_to_produce += + state.expected_total % config->producer_cnt; + args->delay_ms = config->producer_delay_ms; + for (t = 0; t < config->topic_cnt; t++) + args->partitions[t] = config->partitions[t]; + + if (thrd_create(&producer_threads[p], producer_thread_func, + args) != thrd_success) { + TEST_FAIL("Failed to create producer thread %d", p); + } + } + + /* Optional staggered start: give producers a head start before + * spinning up consumer threads. */ + if (config->staggered_start) { + TEST_SAY("Staggered start: producers run alone for 2 s\n"); + rd_sleep(2); + } + + /* Start consumer threads — one per consumer */ + for (c = 0; c < config->consumer_cnt; c++) { + consumer_args[c].state = &state; + consumer_args[c].consumer_id = c; + consumer_args[c].run = rd_true; + consumer_args[c].done = rd_false; + consumer_args[c].poll_delay_ms = config->consumer_delay_ms; + + if (thrd_create(&consumer_threads[c], consumer_thread_func, + &consumer_args[c]) != thrd_success) { + TEST_FAIL("Failed to create consumer thread %d", c); + } + } + + /* Wait for all expected first deliveries, bounded by max_attempts + * ticks of 500ms each (default 240 ticks = 120 s). */ + max_wait_ticks = config->max_attempts > 0 ? config->max_attempts : 240; + wait_ticks = max_wait_ticks; + + while (wait_ticks-- > 0) { + int first_seen; + int consumed_so_far; + int dups; + rd_bool_t producers_done; + + mtx_lock(&state.lock); + first_seen = state.total_first_delivery; + consumed_so_far = state.total_consumed; + dups = state.total_duplicates; + producers_done = state.producers_done; + mtx_unlock(&state.lock); + + if (first_seen >= state.expected_total) + break; + + if (wait_ticks % 20 == 0) + TEST_SAY( + "Progress: consumed %d/%d (first=%d, dup=%d, " + "producers_done=%d)\n", + consumed_so_far, state.expected_total, first_seen, + dups, (int)producers_done); + + rd_usleep(500 * 1000, NULL); + } + + /* Join producers even if drain timed out. */ + for (p = 0; p < config->producer_cnt; p++) + thrd_join(producer_threads[p], NULL); + + /* Tell consumer threads to stop and join. */ + mtx_lock(&state.lock); + for (c = 0; c < config->consumer_cnt; c++) + consumer_args[c].run = rd_false; + mtx_unlock(&state.lock); + + for (c = 0; c < config->consumer_cnt; c++) + thrd_join(consumer_threads[c], NULL); + + /* Snapshot results under lock */ + mtx_lock(&state.lock); + final_produced = state.total_produced; + final_consumed = state.total_consumed; + final_first_delivery = state.total_first_delivery; + final_duplicates = state.total_duplicates; + final_failed = state.test_failed; + mtx_unlock(&state.lock); + + TEST_SAY( + "Final: produced=%d, consumed=%d (first=%d, dup=%d), " + "expected=%d\n", + final_produced, final_consumed, final_first_delivery, + final_duplicates, state.expected_total); + + TEST_ASSERT(final_produced == state.expected_total, + "Expected to produce %d, actually produced %d", + state.expected_total, final_produced); + + /* A record can be redelivered if the broker-side acquisition lock + * expires before an ACCEPT arrives. We distinguish first + * deliveries (delivery_count == 1) from redeliveries via + * rd_kafka_message_delivery_count() and assert strict equality on + * first deliveries — every produced record must have been + * delivered at least once. Duplicates are tracked separately so a + * regression in delivery semantics still shows up in the + * diagnostic output. */ + TEST_ASSERT(final_first_delivery == state.expected_total, + "Expected %d first-time deliveries, got %d " + "(consumed=%d, duplicates=%d)", + state.expected_total, final_first_delivery, final_consumed, + final_duplicates); + TEST_ASSERT(!final_failed, "Producer thread reported failure"); + + TEST_SAY("SUCCESS: %s\n", unique_test_name); + + mtx_destroy(&state.lock); +} + +/*************************************************************************** + * Test Cases - Basic Concurrency + ***************************************************************************/ + +/** + * @brief Single producer, single consumer, single topic + */ +static void do_test_1p_1c_1t_1part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 1, + .producer_cnt = 1, + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 5000, + .group_name = "share-conc-1p1c1t1p", + .test_name = "1 producer, 1 consumer, 1 topic, 1 partition"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Single producer, multiple consumers, single topic + */ +static void do_test_1p_4c_1t_4part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 1, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 5000, + .group_name = "share-conc-1p4c1t4p", + .test_name = "1 producer, 4 consumers, 1 topic, 4 partitions"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Multiple producers, single consumer + */ +static void do_test_4p_1c_1t_4part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 1, + .producer_cnt = 4, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 1250, + .group_name = "share-conc-4p1c1t4p", + .test_name = "4 producers, 1 consumer, 1 topic, 4 partitions"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Multiple producers, multiple consumers, single topic + */ +static void do_test_4p_4c_1t_4part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 4, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 5000, + .group_name = "share-conc-4p4c1t4p", + .test_name = "4 producers, 4 consumers, 1 topic, 4 partitions"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - Multi-Topic Concurrency + ***************************************************************************/ + +/** + * @brief Multiple topics with concurrent access + */ +static void do_test_2p_2c_3t_2part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 2, + .producer_cnt = 2, + .topic_cnt = 3, + .partitions = {2, 2, 2}, + .msgs_per_partition = 1666, + .group_name = "share-conc-2p2c3t2p", + .test_name = + "2 producers, 2 consumers, 3 topics, 2 partitions each"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Many topics with single partition each + */ +static void do_test_2p_2c_8t_1part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 2, + .producer_cnt = 2, + .topic_cnt = 8, + .partitions = {1, 1, 1, 1, 1, 1, 1, 1}, + .msgs_per_partition = 1250, + .group_name = "share-conc-2p2c8t1p", + .test_name = "2 producers, 2 consumers, 8 topics, 1 partition each", + .max_attempts = 150}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - High Contention + ***************************************************************************/ + +/** + * @brief Many consumers competing for single partition + */ +static void do_test_1p_8c_1t_1part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 8, + .producer_cnt = 1, + .topic_cnt = 1, + .partitions = {1}, + .msgs_per_partition = 10000, + .group_name = "share-conc-1p8c1t1p", + .test_name = + "1 producer, 8 consumers, 1 partition (high contention)"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief More consumers than partitions + */ +static void do_test_2p_6c_1t_2part(void) { + concurrent_test_config_t config = { + .consumer_cnt = 6, + .producer_cnt = 2, + .topic_cnt = 1, + .partitions = {2}, + .msgs_per_partition = 15000, + .group_name = "share-conc-2p6c1t2p", + .test_name = + "2 producers, 6 consumers, 2 partitions (3:1 consumer ratio)"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - Explicit Acknowledgement + ***************************************************************************/ + +/** + * @brief Concurrent with explicit acknowledgement + */ +static void do_test_explicit_ack_4p_4c(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 4, + .topic_cnt = 2, + .partitions = {2, 2}, + .msgs_per_partition = 5000, + .group_name = "share-conc-explicit-4p4c", + .test_name = "4 producers, 4 consumers, explicit ack", + .explicit_ack = rd_true}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - Staggered Start + ***************************************************************************/ + +/** + * @brief Consumers start before producers + */ +static void do_test_staggered_start(void) { + concurrent_test_config_t config = { + .consumer_cnt = 2, + .producer_cnt = 2, + .topic_cnt = 1, + .partitions = {2}, + .msgs_per_partition = 5000, + .group_name = "share-conc-staggered", + .test_name = "Staggered start: consumers before producers", + .staggered_start = rd_true}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - High Volume Stress + ***************************************************************************/ + +/** + * @brief High volume concurrent test + */ +static void do_test_high_volume_20k(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 4, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 5000, + .group_name = "share-conc-highvol-20k", + .test_name = "High volume: 4p x 4c x 20k messages", + .max_attempts = 150}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief High volume with many partitions + */ +static void do_test_high_volume_many_partitions(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 4, + .topic_cnt = 1, + .partitions = {8}, + .msgs_per_partition = 2500, + .group_name = "share-conc-highvol-8p", + .test_name = "High volume: 8 partitions x 2.5k messages each", + .max_attempts = 150}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - Asymmetric Configurations + ***************************************************************************/ + +/** + * @brief Many producers, few consumers + */ +static void do_test_8p_2c(void) { + concurrent_test_config_t config = { + .consumer_cnt = 2, + .producer_cnt = 8, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 2500, + .group_name = "share-conc-8p2c", + .test_name = "8 producers, 2 consumers (producer heavy)"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Few producers, many consumers + */ +static void do_test_2p_8c(void) { + concurrent_test_config_t config = { + .consumer_cnt = 8, + .producer_cnt = 2, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 10000, + .group_name = "share-conc-2p8c", + .test_name = "2 producers, 8 consumers (consumer heavy)"}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - Complex Multi-Topic Multi-Partition + ***************************************************************************/ + +/** + * @brief Complex scenario with varied partition counts + */ +static void do_test_complex_varied_partitions(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 4, + .topic_cnt = 4, + .partitions = {1, 2, 3, 4}, + .msgs_per_partition = 2000, + .group_name = "share-conc-complex-varied", + .test_name = "4 topics with 1,2,3,4 partitions respectively", + .max_attempts = 150}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Maximum concurrent scenario + */ +static void do_test_max_concurrent(void) { + concurrent_test_config_t config = { + .consumer_cnt = MAX_CONSUMERS, + .producer_cnt = MAX_PRODUCERS, + .topic_cnt = 4, + .partitions = {4, 4, 4, 4}, + .msgs_per_partition = 3125, + .group_name = "share-conc-max", + .test_name = "Maximum: 10 producers, 10 consumers, 4 topics", + .max_attempts = 200}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/*************************************************************************** + * Test Cases - Slow Consumer/Producer Scenarios + ***************************************************************************/ + +/** + * @brief Slow consumers with fast producers + */ +static void do_test_slow_consumers(void) { + concurrent_test_config_t config = { + .consumer_cnt = 2, + .producer_cnt = 4, + .topic_cnt = 1, + .partitions = {2}, + .msgs_per_partition = 5000, + .group_name = "share-conc-slow-consumer", + .test_name = "Fast producers, slow consumers", + .consumer_delay_ms = 50, + .max_attempts = 200}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Fast consumers with slow producers + */ +static void do_test_slow_producers(void) { + concurrent_test_config_t config = { + .consumer_cnt = 4, + .producer_cnt = 2, + .topic_cnt = 1, + .partitions = {2}, + .msgs_per_partition = 500, + .group_name = "share-conc-slow-producer", + .test_name = "Slow producers, fast consumers", + .producer_delay_ms = 10000, + .max_attempts = 200}; + + SUB_TEST(); + run_concurrent_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Chaos test: consumer churn during sustained production. + * + * Setup: + * - 4 producers in dedicated threads producing fixed total messages. + * - Pool of CONSUMER_POOL slots; only ACTIVE_CONSUMERS run concurrently. + * - Initially start ACTIVE_CONSUMERS threads. + * - For ITERATIONS rounds: sleep ITERATION_MS, then stop a random number + * of the currently running consumers (>= 1) and start the same number + * of fresh ones from the pool. Old threads are joined at the very end. + * - After iterations, drain remaining messages and shut everyone down. + * + * Invariant for share consumers: every produced record is first-delivered + * exactly once. Records that a closing consumer hadn't acked are released + * back to the group and re-delivered with delivery_count > 1 (counted as + * duplicates, not first deliveries). + */ +static void do_test_chaos_consumer_lifecycle(rd_bool_t explicit_ack) { + concurrent_test_state_t state = {0}; + const concurrent_test_config_t config = { + .consumer_cnt = 4, /* informational; pool size below */ + .producer_cnt = 4, + .topic_cnt = 1, + .partitions = {4}, + .msgs_per_partition = 1000, + .group_name = explicit_ack ? "share-chaos-lifecycle-explicit" + : "share-chaos-lifecycle-implicit", + .test_name = explicit_ack ? "Chaos (explicit ack): consumer " + "churn during production" + : "Chaos (implicit ack): consumer " + "churn during production", + .explicit_ack = explicit_ack}; + const int ACTIVE_CONSUMERS = 4; + const int CONSUMER_POOL = 12; + const int ITERATIONS = 3; + const int ITERATION_MS = 6000; + const int join_poll_ticks = 10; /* 10 * 500ms = 5 s */ + thrd_t producer_threads[MAX_PRODUCERS]; + thrd_t consumer_threads[MAX_CONSUMER_POOL]; + consumer_thread_args_t consumer_args[MAX_CONSUMER_POOL]; + int started_threads = 0; + int first_running = 0; + int last_running; + int t, c, p, iter; + int total_partitions = 0; + int msgs_per_producer; + int wait_attempts; + int dangling = 0; + int final_produced; + int final_first_delivery; + int final_duplicates; + int final_consumed; + rd_bool_t final_failed; + rd_kafka_share_t *dummy_consumer; + + SUB_TEST(); + + TEST_ASSERT(CONSUMER_POOL <= MAX_CONSUMER_POOL, + "CONSUMER_POOL %d > MAX_CONSUMER_POOL", CONSUMER_POOL); + TEST_ASSERT(ACTIVE_CONSUMERS <= CONSUMER_POOL, "ACTIVE > POOL"); + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== %s ===\n", config.test_name); + TEST_SAY( + "============================================================\n"); + TEST_SAY("Active %d, pool %d, iterations %d × %d ms\n", + ACTIVE_CONSUMERS, CONSUMER_POOL, ITERATIONS, ITERATION_MS); + + this_test = test_curr; + + /* See note in run_concurrent_test: set these on the main thread + * before spawning worker threads to avoid TSAN-flagged races on + * the shared test struct that test_curr aliases. */ + test_curr->exp_dr_status = (rd_kafka_msg_status_t)-1; + test_curr->ignore_dr_err = rd_true; + + mtx_init(&state.lock, mtx_plain); + state.topic_cnt = config.topic_cnt; + state.group_name = config.group_name; + state.explicit_ack = config.explicit_ack; + state.producers_remaining = config.producer_cnt; + + for (t = 0; t < config.topic_cnt; t++) + total_partitions += config.partitions[t]; + state.expected_total = total_partitions * config.msgs_per_partition; + TEST_SAY("Expected messages: %d\n", state.expected_total); + + for (t = 0; t < config.topic_cnt; t++) { + rd_snprintf(state.topics[t], sizeof(state.topics[t]), "%s", + test_mk_topic_name("0174-chaos", 1)); + test_create_topic_wait_exists( + NULL, state.topics[t], config.partitions[t], -1, 60 * 1000); + } + + /* Apply broker-side group config (auto.offset.reset=earliest) via + * a throwaway consumer registration, so the alter sticks before + * any chaos worker subscribes. */ + dummy_consumer = test_create_share_consumer(config.group_name, NULL); + test_share_set_auto_offset_reset(config.group_name, "earliest"); + test_share_consumer_close(dummy_consumer); + test_share_destroy(dummy_consumer); + + /* Producer threads */ + msgs_per_producer = state.expected_total / config.producer_cnt; + for (p = 0; p < config.producer_cnt; p++) { + producer_thread_args_t *args = rd_calloc(1, sizeof(*args)); + args->state = &state; + args->producer_id = p; + args->msgs_to_produce = msgs_per_producer; + if (p == config.producer_cnt - 1) + args->msgs_to_produce += + state.expected_total % config.producer_cnt; + /* Light pacing so production stretches across the chaos + * iterations rather than finishing before the first churn. */ + args->delay_ms = 5; + for (t = 0; t < config.topic_cnt; t++) + args->partitions[t] = config.partitions[t]; + + if (thrd_create(&producer_threads[p], producer_thread_func, + args) != thrd_success) { + TEST_FAIL("Failed to create producer thread %d", p); + } + } + + /* Pre-init the entire pool */ + for (c = 0; c < CONSUMER_POOL; c++) { + consumer_args[c].state = &state; + consumer_args[c].consumer_id = c; + consumer_args[c].run = rd_false; + consumer_args[c].done = rd_false; + consumer_args[c].poll_delay_ms = 0; + } + + /* Start the initial active set */ + for (c = 0; c < ACTIVE_CONSUMERS; c++) { + mtx_lock(&state.lock); + consumer_args[c].run = rd_true; + mtx_unlock(&state.lock); + if (thrd_create(&consumer_threads[c], consumer_thread_func, + &consumer_args[c]) != thrd_success) { + TEST_FAIL( + "Failed to create initial consumer " + "thread %d", + c); + } + started_threads++; + } + last_running = ACTIVE_CONSUMERS - 1; + + srand((unsigned int)time(NULL)); + + for (iter = 0; iter < ITERATIONS; iter++) { + int running; + int churn; + int can_start; + int s; + + TEST_SAY("--- Iteration %d/%d: sleeping %d ms ---\n", iter + 1, + ITERATIONS, ITERATION_MS); + rd_usleep(ITERATION_MS * 1000, NULL); + + running = last_running - first_running + 1; + /* Stop at least 1, at most floor(running/2) for stability */ + churn = (running > 1) ? (rand() % ((running / 2) + 1) + 1) : 1; + TEST_SAY("Iteration %d: stopping %d/%d running consumers\n", + iter + 1, churn, running); + + mtx_lock(&state.lock); + for (s = 0; s < churn && first_running <= last_running; + s++, first_running++) { + consumer_args[first_running].run = rd_false; + } + mtx_unlock(&state.lock); + + /* Replenish with new pool slots (don't reuse stopped slots) */ + can_start = CONSUMER_POOL - started_threads; + if (can_start <= 0) { + TEST_SAY( + "Pool exhausted (%d threads spawned); no more " + "starts this iteration\n", + started_threads); + continue; + } + if (churn > can_start) + churn = can_start; + + TEST_SAY("Iteration %d: starting %d fresh consumers\n", + iter + 1, churn); + + for (s = 0; s < churn; s++) { + int idx = started_threads; + if (idx >= CONSUMER_POOL) + break; + mtx_lock(&state.lock); + consumer_args[idx].run = rd_true; + mtx_unlock(&state.lock); + if (thrd_create(&consumer_threads[idx], + consumer_thread_func, + &consumer_args[idx]) != thrd_success) { + TEST_FAIL( + "Failed to create chaos consumer " + "thread %d", + idx); + } + started_threads++; + last_running = idx; + } + } + + TEST_SAY( + "Iterations done; waiting for production to drain, then " + "shutting down consumers\n"); + + /* Drain loop, bounded so we don't sit for minutes on a stuck + * test: + * - 60 ticks of 500 ms = 30 s absolute cap + * - early exit when producers_done AND first_delivery has not + * advanced for 10 consecutive ticks (~5 s of no progress). + * If the chaos test has stalled, we report and move on + * rather than waiting for the full window. */ + wait_attempts = 60; /* ~30 seconds absolute cap */ + int prev_first_seen = -1; + int stale_ticks = 0; + const int stale_limit = 10; /* ~5 s with no progress */ + + while (wait_attempts-- > 0) { + int first_seen; + rd_bool_t producers_done; + + mtx_lock(&state.lock); + first_seen = state.total_first_delivery; + producers_done = state.producers_done; + mtx_unlock(&state.lock); + + if (producers_done && first_seen >= state.expected_total) + break; + + if (first_seen == prev_first_seen) { + stale_ticks++; + if (producers_done && stale_ticks >= stale_limit) { + TEST_SAY( + "Drain: no progress for " + "~%d s after producers done; " + "exiting at %d/%d " + "(test will fail in assertion)\n", + stale_ticks / 2, first_seen, + state.expected_total); + break; + } + } else { + stale_ticks = 0; + prev_first_seen = first_seen; + } + + if (wait_attempts % 10 == 0) + TEST_SAY( + "Drain: %d/%d first deliveries " + "(producers_done=%d)\n", + first_seen, state.expected_total, + (int)producers_done); + + rd_usleep(500 * 1000, NULL); + } + + /* Join producers - these always exit cleanly so a hard join is + * safe. */ + for (p = 0; p < config.producer_cnt; p++) + thrd_join(producer_threads[p], NULL); + + /* Stop all consumer threads. */ + mtx_lock(&state.lock); + for (c = 0; c < started_threads; c++) + consumer_args[c].run = rd_false; + mtx_unlock(&state.lock); + + /* Bounded wait for consumers to exit cleanly. A wedged consumer + * (eg blocked inside consume_batch waiting on a stale share + * session) won't return from rd_kafka_share_consumer_close, so + * thrd_join would block indefinitely. We poll each consumer's + * done flag for up to ~5 s; any thread that hasn't set it by + * then is left dangling for the OS to reclaim at process exit + * and we move straight to the assertions. This matters only + * when the test is failing — once the underlying librdkafka + * issue is fixed, every consumer thread reaches its `done = true` + * within tens of milliseconds. */ + for (c = 0; c < started_threads; c++) { + rd_bool_t done = rd_false; + for (t = 0; t < join_poll_ticks; t++) { + mtx_lock(&state.lock); + done = consumer_args[c].done; + mtx_unlock(&state.lock); + if (done) + break; + rd_usleep(500 * 1000, NULL); + } + if (done) { + thrd_join(consumer_threads[c], NULL); + } else { + /* Detach so the C runtime won't leak the + * thrd_t handle when the process exits; + * the OS will tear down the still-running + * thread when the process terminates. */ + thrd_detach(consumer_threads[c]); + dangling++; + TEST_SAY( + "Consumer thread %d: did not exit within " + "5 s of stop signal — detaching and " + "proceeding (likely wedged inside " + "librdkafka close path)\n", + consumer_args[c].consumer_id); + } + } + + if (dangling > 0) + TEST_SAY( + "WARNING: %d/%d consumer thread(s) left " + "dangling at test exit\n", + dangling, started_threads); + + mtx_lock(&state.lock); + final_produced = state.total_produced; + final_first_delivery = state.total_first_delivery; + final_duplicates = state.total_duplicates; + final_consumed = state.total_consumed; + final_failed = state.test_failed; + mtx_unlock(&state.lock); + + TEST_SAY( + "Chaos final: produced=%d, consumed=%d (first=%d, dup=%d), " + "expected=%d, threads spawned=%d\n", + final_produced, final_consumed, final_first_delivery, + final_duplicates, state.expected_total, started_threads); + + TEST_ASSERT(final_produced == state.expected_total, + "Expected to produce %d, actually produced %d", + state.expected_total, final_produced); + TEST_ASSERT(final_first_delivery == state.expected_total, + "Expected %d first-time deliveries, got %d " + "(consumed=%d, duplicates=%d)", + state.expected_total, final_first_delivery, final_consumed, + final_duplicates); + TEST_ASSERT(!final_failed, "Producer thread reported failure"); + TEST_SAY( + "SUCCESS: chaos test - %d threads churned, " + "every record first-delivered exactly once\n", + started_threads); + + mtx_destroy(&state.lock); + + SUB_TEST_PASS(); +} + + +/*************************************************************************** + * Main Entry Point + ***************************************************************************/ + +int main_0174_share_consumer_concurrency(int argc, char **argv) { + + test_timeout_set(1500); + + /* Every test below runs producers AND consumers in dedicated + * threads via run_concurrent_test(). There is no main-thread + * polling — consumer_thread_func handles the poll loop per + * consumer. */ + + /* Basic concurrency tests */ + do_test_1p_1c_1t_1part(); /* Baseline: 1 producer, 1 consumer */ + do_test_1p_4c_1t_4part(); /* Fan out: 1 producer, 4 consumers */ + do_test_4p_1c_1t_4part(); /* Fan in: 4 producers, 1 consumer */ + do_test_4p_4c_1t_4part(); /* Symmetric: 4 producers, 4 consumers */ + + /* Multi-topic tests */ + do_test_2p_2c_3t_2part(); /* Multiple topics */ + do_test_2p_2c_8t_1part(); /* Many topics */ + + /* High contention tests */ + do_test_1p_8c_1t_1part(); /* Many consumers, 1 partition */ + do_test_2p_6c_1t_2part(); /* More consumers than partitions */ + + /* Explicit acknowledgement */ + do_test_explicit_ack_4p_4c(); + + /* Staggered start */ + do_test_staggered_start(); + + /* High volume stress tests */ + do_test_high_volume_20k(); + do_test_high_volume_many_partitions(); + + /* Asymmetric configurations */ + do_test_8p_2c(); /* Producer heavy */ + do_test_2p_8c(); /* Consumer heavy */ + + /* Complex scenarios */ + do_test_complex_varied_partitions(); + do_test_max_concurrent(); + + /* Slow consumer/producer scenarios */ + do_test_slow_consumers(); + do_test_slow_producers(); + + /* Chaos test: consumer lifecycle churn during sustained production. + * Run once with explicit acknowledgement and once with implicit + * acknowledgement to cover both ack flows under churn. */ + do_test_chaos_consumer_lifecycle(rd_true); + do_test_chaos_consumer_lifecycle(rd_false); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0175-share_consumer_groups.c b/lib/librdkafka-2.15.0/tests/0175-share_consumer_groups.c new file mode 100644 index 00000000000..1881fbb0cb3 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0175-share_consumer_groups.c @@ -0,0 +1,574 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "rdkafka.h" + +/** + * @name Share Consumer Multiple Groups Tests + * + * Tests for multiple share groups consuming from the same topic independently. + * Each share group should receive all messages independently of other groups. + */ + +#define MAX_GROUPS 8 +#define MAX_CONSUMERS_PER_GRP 4 +#define MAX_PARTITIONS 8 +#define BATCH_SIZE 1000 + +/** + * @brief Configuration for a multi-group share consumer test + */ +typedef struct { + int group_cnt; /**< Number of share groups */ + int consumers_per_group[MAX_GROUPS]; /**< Consumers per group */ + const char *group_names[MAX_GROUPS]; /**< Group names */ + rd_bool_t use_earliest[MAX_GROUPS]; /**< Use earliest offset */ + int partitions; /**< Number of partitions */ + int msgs_per_partition; /**< Messages per partition */ + const char *test_name; /**< Test description */ + int max_attempts; /**< Max poll attempts */ + rd_bool_t produce_before_subscribe; /**< Produce before subscribe */ + int msgs_produce_after_subscribe; /**< Messages after subscribe */ +} groups_test_config_t; + +/** + * @brief Test state for multi-group tests + */ +typedef struct { + rd_kafka_share_t *consumers[MAX_GROUPS][MAX_CONSUMERS_PER_GRP]; + int consumed[MAX_GROUPS]; + int expected_per_group; + char *topic; +} groups_test_state_t; + +/** + * @brief Create consumers for all groups + */ +static void create_group_consumers(groups_test_config_t *config, + groups_test_state_t *state) { + int g, c; + + for (g = 0; g < config->group_cnt; g++) { + for (c = 0; c < config->consumers_per_group[g]; c++) { + state->consumers[g][c] = test_create_share_consumer( + config->group_names[g], NULL); + } + + /* Configure group offset if earliest */ + if (config->use_earliest[g]) { + test_share_set_auto_offset_reset(config->group_names[g], + "earliest"); + } + + TEST_SAY("Created %d consumer(s) for group '%s' (offset=%s)\n", + config->consumers_per_group[g], config->group_names[g], + config->use_earliest[g] ? "earliest" : "latest"); + } +} + +/** + * @brief Setup topic and produce initial messages + */ +static void setup_topic_and_produce(groups_test_config_t *config, + groups_test_state_t *state, + int msg_cnt) { + int p; + + state->topic = rd_strdup(test_mk_topic_name("0175-groups", 1)); + test_create_topic_wait_exists(NULL, state->topic, config->partitions, + -1, 60 * 1000); + + if (msg_cnt > 0) { + for (p = 0; p < config->partitions; p++) { + test_produce_msgs_easy(state->topic, 0, p, + msg_cnt / config->partitions); + } + TEST_SAY("Produced %d messages to topic '%s' (%d partitions)\n", + msg_cnt, state->topic, config->partitions); + } +} + +/** + * @brief Subscribe all consumers to the topic + */ +static void subscribe_all_consumers(groups_test_config_t *config, + groups_test_state_t *state) { + rd_kafka_topic_partition_list_t *subs; + int g, c; + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, state->topic, + RD_KAFKA_PARTITION_UA); + + for (g = 0; g < config->group_cnt; g++) { + for (c = 0; c < config->consumers_per_group[g]; c++) { + rd_kafka_share_subscribe(state->consumers[g][c], subs); + } + } + + rd_kafka_topic_partition_list_destroy(subs); + TEST_SAY("Subscribed all consumers to topic '%s'\n", state->topic); + + /* Allow consumers to join groups */ + rd_sleep(2); +} + +/** + * @brief Consume messages from all groups + */ +static void consume_from_all_groups(groups_test_config_t *config, + groups_test_state_t *state) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + int attempts; + int g, c; + int idle_rounds = 0; + int round_consumed; + int all_done; + size_t rcvd; + size_t m; + char progress[256]; + int pos; + + for (g = 0; g < config->group_cnt; g++) { + state->consumed[g] = 0; + } + + attempts = config->max_attempts > 0 ? config->max_attempts : 100; + + while (attempts-- > 0 && idle_rounds < 20) { + round_consumed = 0; + all_done = 1; + + for (g = 0; g < config->group_cnt; g++) { + if (state->consumed[g] >= state->expected_per_group) + continue; + + all_done = 0; + + for (c = 0; c < config->consumers_per_group[g]; c++) { + err = rd_kafka_share_poll( + state->consumers[g][c], 500, &batch); + if (err) { + TEST_SAY( + "Group %d consumer %d: " + "share_poll failed: %s\n", + g, c, rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (msg && !msg->err) + state->consumed[g]++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + round_consumed += (int)rcvd; + } + } + + if (all_done) + break; + + if (round_consumed > 0) { + idle_rounds = 0; + if (attempts % 10 == 0) { + memset(progress, 0, sizeof(progress)); + pos = 0; + for (g = 0; g < config->group_cnt && pos < 200; + g++) { + pos += rd_snprintf( + progress + pos, + sizeof(progress) - pos, + "G%d=%d/%d ", g, state->consumed[g], + state->expected_per_group); + } + TEST_SAY("Progress: %s\n", progress); + } + } else { + idle_rounds++; + } + } +} + +/** + * @brief Cleanup test resources + */ +static void cleanup_groups_test(groups_test_config_t *config, + groups_test_state_t *state) { + int g, c; + + if (state->topic) { + test_delete_topic( + test_share_consumer_get_rk(state->consumers[0][0]), + state->topic); + rd_free(state->topic); + } + + for (g = 0; g < config->group_cnt; g++) { + for (c = 0; c < config->consumers_per_group[g]; c++) { + if (state->consumers[g][c]) { + test_share_consumer_close( + state->consumers[g][c]); + test_share_destroy(state->consumers[g][c]); + } + } + } +} + +/** + * @brief Run a multi-group test + */ +static void run_groups_test(groups_test_config_t *config) { + groups_test_state_t state = {0}; + int g; + int p; + int per_partition; + int total_msgs; + char result[256] = {0}; + int pos = 0; + char unique_suffix[64]; + char unique_test_name[256]; + char unique_group_names[MAX_GROUPS][128]; + + /* Per-invocation unique suffix; appended to the test name and + * to every group name so re-runs / parallel runs get fresh + * share-groups on the broker. */ + rd_snprintf(unique_suffix, sizeof(unique_suffix), "rnd%" PRIx64, + test_id_generate()); + rd_snprintf(unique_test_name, sizeof(unique_test_name), "%s [%s]", + config->test_name, unique_suffix); + config->test_name = unique_test_name; + for (g = 0; g < config->group_cnt; g++) { + rd_snprintf(unique_group_names[g], + sizeof(unique_group_names[g]), "%s-%s", + config->group_names[g], unique_suffix); + config->group_names[g] = unique_group_names[g]; + } + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== %s ===\n", config->test_name); + TEST_SAY( + "============================================================\n"); + + /* Calculate expected messages */ + total_msgs = config->partitions * config->msgs_per_partition; + if (config->produce_before_subscribe) { + state.expected_per_group = total_msgs; + } else { + state.expected_per_group = 0; + } + state.expected_per_group += config->msgs_produce_after_subscribe; + + /* Create consumers */ + create_group_consumers(config, &state); + + /* Setup topic and produce initial messages if configured */ + if (config->produce_before_subscribe) { + setup_topic_and_produce(config, &state, total_msgs); + } else { + setup_topic_and_produce(config, &state, 0); + } + + /* Subscribe all consumers */ + subscribe_all_consumers(config, &state); + + /* Produce after subscribe if configured */ + if (config->msgs_produce_after_subscribe > 0) { + per_partition = + config->msgs_produce_after_subscribe / config->partitions; + for (p = 0; p < config->partitions; p++) { + test_produce_msgs_easy(state.topic, 0, p, + per_partition); + } + TEST_SAY("Produced %d messages after subscribe\n", + config->msgs_produce_after_subscribe); + } + + /* Consume from all groups */ + consume_from_all_groups(config, &state); + + /* Verify results */ + for (g = 0; g < config->group_cnt; g++) { + pos += rd_snprintf(result + pos, sizeof(result) - pos, + "G%d=%d ", g, state.consumed[g]); + } + TEST_SAY("Results: %s(expected %d each)\n", result, + state.expected_per_group); + + for (g = 0; g < config->group_cnt; g++) { + /* Allow some tolerance for latest offset tests */ + if (!config->use_earliest[g]) { + TEST_ASSERT( + state.consumed[g] >= state.expected_per_group - 5 && + state.consumed[g] <= + state.expected_per_group + 5, + "Group %d expected ~%d messages, got %d", g, + state.expected_per_group, state.consumed[g]); + } else { + TEST_ASSERT( + state.consumed[g] == state.expected_per_group, + "Group %d expected %d messages, got %d", g, + state.expected_per_group, state.consumed[g]); + } + } + + TEST_SAY("SUCCESS: %s\n", config->test_name); + + cleanup_groups_test(config, &state); +} + +/*************************************************************************** + * Test Cases + ***************************************************************************/ + +/** + * @brief Two groups consuming same topic independently + */ +static void do_test_two_groups_same_topic(void) { + groups_test_config_t config = { + .group_cnt = 2, + .consumers_per_group = {1, 1}, + .group_names = {"share-2grp-A", "share-2grp-B"}, + .use_earliest = {rd_true, rd_true}, + .partitions = 1, + .msgs_per_partition = 100, + .test_name = "Two groups consuming same topic", + .produce_before_subscribe = rd_true, + }; + + SUB_TEST(); + run_groups_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Three groups with 2 consumers each + */ +static void do_test_three_groups_concurrent(void) { + groups_test_config_t config = { + .group_cnt = 3, + .consumers_per_group = {2, 2, 2}, + .group_names = {"share-3grp-A", "share-3grp-B", "share-3grp-C"}, + .use_earliest = {rd_true, rd_true, rd_true}, + .partitions = 3, + .msgs_per_partition = 334, + .test_name = "Three groups with 2 consumers each", + .produce_before_subscribe = rd_true, + .max_attempts = 150, + }; + + SUB_TEST(); + run_groups_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Five groups consuming same topic + */ +static void do_test_five_groups_same_topic(void) { + groups_test_config_t config = { + .group_cnt = 5, + .consumers_per_group = {1, 1, 1, 1, 1}, + .group_names = {"share-5grp-A", "share-5grp-B", "share-5grp-C", + "share-5grp-D", "share-5grp-E"}, + .use_earliest = {rd_true, rd_true, rd_true, rd_true, rd_true}, + .partitions = 2, + .msgs_per_partition = 250, + .test_name = "Five groups consuming same topic", + .produce_before_subscribe = rd_true, + .max_attempts = 150, + }; + + SUB_TEST(); + run_groups_test(&config); + + SUB_TEST_PASS(); +} + +/** + * @brief Groups joining at staggered times + */ +static void do_test_groups_staggered_join(void) { + rd_kafka_share_t *consumer_a, *consumer_b; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + const char *topic; + const char *group_a = "share-stagger-A"; + const char *group_b = "share-stagger-B"; + const int msg_cnt = 100; + int consumed_a = 0, consumed_b = 0; + int attempts; + size_t rcvd; + size_t m; + + SUB_TEST(); + + TEST_SAY("\n"); + TEST_SAY( + "============================================================\n"); + TEST_SAY("=== Groups joining at staggered times ===\n"); + TEST_SAY( + "============================================================\n"); + + /* Create topic */ + topic = test_mk_topic_name("0175-staggered", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Create first consumer */ + consumer_a = test_create_share_consumer(group_a, NULL); + test_share_set_auto_offset_reset(group_a, "earliest"); + + /* Produce messages */ + test_produce_msgs_easy(topic, 0, 0, msg_cnt); + + /* Subscribe first consumer */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(consumer_a, subs); + rd_sleep(2); + + /* Group A consumes half */ + attempts = 30; + while (consumed_a < msg_cnt / 2 && attempts-- > 0) { + err = rd_kafka_share_poll(consumer_a, 1000, &batch); + if (err) { + TEST_SAY("Group A: share_poll failed: %s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (msg && !msg->err) + consumed_a++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_SAY("Group A consumed %d so far\n", consumed_a); + + /* Now create second consumer */ + consumer_b = test_create_share_consumer(group_b, NULL); + test_share_set_auto_offset_reset(group_b, "earliest"); + rd_kafka_share_subscribe(consumer_b, subs); + rd_kafka_topic_partition_list_destroy(subs); + rd_sleep(2); + + /* Both finish consuming */ + attempts = 50; + while ((consumed_a < msg_cnt || consumed_b < msg_cnt) && + attempts-- > 0) { + if (consumed_a < msg_cnt) { + err = rd_kafka_share_poll(consumer_a, 500, &batch); + if (!err) { + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (msg && !msg->err) + consumed_a++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } else { + TEST_SAY( + "Group A: share_poll failed: " + "%s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + } + } + + if (consumed_b < msg_cnt) { + err = rd_kafka_share_poll(consumer_b, 500, &batch); + if (!err) { + rcvd = rd_kafka_messages_count(batch); + for (m = 0; m < rcvd; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + if (msg && !msg->err) + consumed_b++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } else { + TEST_SAY( + "Group B: share_poll failed: " + "%s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + } + } + } + + TEST_SAY("Results: A=%d, B=%d (expected %d each)\n", consumed_a, + consumed_b, msg_cnt); + + TEST_ASSERT(consumed_a == msg_cnt, "Group A expected %d, got %d", + msg_cnt, consumed_a); + TEST_ASSERT(consumed_b == msg_cnt, "Group B expected %d, got %d", + msg_cnt, consumed_b); + + TEST_SAY("SUCCESS: Groups joining at staggered times\n"); + + test_share_consumer_close(consumer_a); + test_share_consumer_close(consumer_b); + test_delete_topic(test_share_consumer_get_rk(consumer_a), topic); + test_share_destroy(consumer_a); + test_share_destroy(consumer_b); + + SUB_TEST_PASS(); +} + +int main_0175_share_consumer_groups(int argc, char **argv) { + + /* Basic multiple groups tests */ + do_test_two_groups_same_topic(); + do_test_three_groups_concurrent(); + + /* Scale tests */ + do_test_five_groups_same_topic(); + + /* Timing tests */ + do_test_groups_staggered_join(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0176-share_consumer_commit_sync.c b/lib/librdkafka-2.15.0/tests/0176-share_consumer_commit_sync.c new file mode 100644 index 00000000000..3589fd67a71 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0176-share_consumer_commit_sync.c @@ -0,0 +1,2986 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @brief Share consumer rd_kafka_share_commit_sync() API tests. + * + * Tests the commit_sync API in both implicit and explicit ack modes. + * Verifies that commit_sync synchronously commits acknowledged records + * and returns per-partition results. + */ + +#define CONSUME_ARRAY 10001 + +/** Common producer reused across all non-mock subtests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all non-mock subtests. */ +static rd_kafka_t *common_admin; + + +/** + * @brief Create share consumer with specified ack mode. + * @param ack_mode "implicit" or "explicit" + */ +static rd_kafka_share_t *create_share_consumer(const char *group_id, + const char *ack_mode) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + + rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "share.acknowledgement.mode", ack_mode, errstr, + sizeof(errstr)); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + + return rkshare; +} + + +/** + * @brief Set share.record.lock.duration.ms for a share group. + */ +static void set_group_lock_duration(const char *group_name, + const char *duration_ms) { + const char *cfg[] = {"share.record.lock.duration.ms", "SET", + duration_ms}; + + test_IncrementalAlterConfigs_simple( + common_admin, RD_KAFKA_RESOURCE_GROUP, group_name, cfg, 1); +} + + +/** + * @brief Subscribe a share consumer to topics. + */ +static void subscribe_consumer(rd_kafka_share_t *rkshare, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + int i; + + subs = rd_kafka_topic_partition_list_new(topic_cnt); + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(subs, topics[i], + RD_KAFKA_PARTITION_UA); + + err = rd_kafka_share_subscribe(rkshare, subs); + TEST_ASSERT(!err, "Subscribe failed: %s", rd_kafka_err2str(err)); + + rd_kafka_topic_partition_list_destroy(subs); +} + + +/* =================================================================== + * Test 1: Basic implicit ack mode commit_sync. + * + * Implicit mode auto-ACCEPTs all records from previous poll. + * Consumer consumes records, calls commit_sync, verifies + * per-partition results show NO_ERROR. + * =================================================================== */ +static void do_test_basic_implicit_commit_sync(void) { + const char *topic; + const char *group = "commit-sync-implicit-basic"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-impl-basic", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 5); + + rkshare = create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume records */ + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumed %d messages\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 messages, got %d", consumed); + + /* commit_sync should flush implicit acks */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT(partitions != NULL, + "Expected per-partition results, got NULL"); + + TEST_SAY("commit_sync returned %d partition(s)\n", partitions->cnt); + TEST_ASSERT(partitions->cnt == 1, + "Expected exactly 1 partition result, got %d", + partitions->cnt); + TEST_SAY(" %s [%" PRId32 "]: %s\n", partitions->elems[0].topic, + partitions->elems[0].partition, + rd_kafka_err2str(partitions->elems[0].err)); + TEST_ASSERT(!strcmp(partitions->elems[0].topic, topic), + "Expected topic %s, got %s", topic, + partitions->elems[0].topic); + TEST_ASSERT(partitions->elems[0].partition == 0, + "Expected partition 0, got %" PRId32, + partitions->elems[0].partition); + TEST_ASSERT(partitions->elems[0].err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR, got %s", + rd_kafka_err2str(partitions->elems[0].err)); + + rd_kafka_topic_partition_list_destroy(partitions); + + /* Wait for acquisition lock to expire (3s + 1s buffer) */ + rd_sleep(4); + + /* Produce 5 verification records and consume with the same + * consumer — should get only these 5 (dc == 1), proving + * the first batch was committed and not redelivered. */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + consumed = 0; + attempts = 0; + while (consumed < 5 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 5000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT( + rd_kafka_message_delivery_count(rkm) == 1, + "Got redelivered record at offset %" PRId64 + " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Verification: got %d messages (expected 5)\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 verification records, got %d", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 2: Basic explicit ack mode commit_sync. + * + * Explicit mode requires the app to ACCEPT each record via + * rd_kafka_share_acknowledge(). Then commit_sync flushes acks + * and returns per-partition results. + * =================================================================== */ +static void do_test_basic_explicit_commit_sync(void) { + const char *topic; + const char *group = "commit-sync-explicit-basic"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-expl-basic", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 5); + + rkshare = create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + set_group_lock_duration(group, "3000"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume records and explicitly ACCEPT each */ + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumed and acknowledged %d messages\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 messages, got %d", consumed); + + /* commit_sync should flush explicit acks */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT(partitions != NULL, + "Expected per-partition results, got NULL"); + + TEST_SAY("commit_sync returned %d partition(s)\n", partitions->cnt); + TEST_ASSERT(partitions->cnt == 1, + "Expected exactly 1 partition result, got %d", + partitions->cnt); + TEST_SAY(" %s [%" PRId32 "]: %s\n", partitions->elems[0].topic, + partitions->elems[0].partition, + rd_kafka_err2str(partitions->elems[0].err)); + TEST_ASSERT(!strcmp(partitions->elems[0].topic, topic), + "Expected topic %s, got %s", topic, + partitions->elems[0].topic); + TEST_ASSERT(partitions->elems[0].partition == 0, + "Expected partition 0, got %" PRId32, + partitions->elems[0].partition); + TEST_ASSERT(partitions->elems[0].err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR, got %s", + rd_kafka_err2str(partitions->elems[0].err)); + + rd_kafka_topic_partition_list_destroy(partitions); + + /* Wait for acquisition lock to expire (3s + 1s buffer) */ + rd_sleep(4); + + /* Produce 5 verification records and consume with the same + * consumer — should get only these 5 (dc == 1), proving + * the first batch was committed and not redelivered. */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + consumed = 0; + attempts = 0; + while (consumed < 5 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 5000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT( + rd_kafka_message_delivery_count(rkm) == 1, + "Got redelivered record at offset %" PRId64 + " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Verification: got %d messages (expected 5)\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 verification records, got %d", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 3: No pending acks — commit_sync with nothing to commit. + * + * Subscribe but do not consume any records. commit_sync should + * return NULL error and NULL partitions. + * =================================================================== */ +static void do_test_no_pending_acks(void) { + const char *topic; + const char *group = "commit-sync-no-pending"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-no-pending", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + + rkshare = create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* commit_sync with no consumed records */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT(partitions == NULL, + "Expected NULL partitions when no acks pending, " + "got %d partition(s)", + partitions ? partitions->cnt : -1); + + TEST_SAY( + "commit_sync with no pending acks returned NULL as expected\n"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 4: commit_sync prevents redelivery. + * + * Consumer A consumes and commit_sync's all records, then closes. + * Consumer B (same group) should see 0 records since they were + * all committed by A. + * =================================================================== */ +static void do_test_commit_sync_prevents_redelivery(void) { + const char *topic; + const char *group = "commit-sync-no-redeliver"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-no-redeliver", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Consumer A: consume all and commit_sync */ + rkshare = create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumer A consumed %d messages\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5, got %d", consumed); + + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE(partitions, rd_kafka_topic_partition_list_destroy); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Consumer B: should only get the 5 verification records. + * Close tears down the connection and broker releases + * records immediately — no lock wait needed. */ + rkshare = create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer B share_poll returned: rcvd=%zu, error=%s\n", rcvd, + error ? rd_kafka_error_string(error) : "none"); + if (error) + rd_kafka_error_destroy(error); + + consumed = 0; + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer B got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer B got %d messages (expected 5)\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 verification records, got %d", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 5: Mixed ack types — ACCEPT, RELEASE, REJECT. + * + * Consumer A: ACCEPT first 5, RELEASE next 3, REJECT last 2, + * then commit_sync. Consumer B should only receive the 3 + * RELEASE'd records with delivery_count >= 2. + * =================================================================== */ +static void do_test_mixed_ack_types(void) { + const char *topic; + const char *group = "commit-sync-mixed-acks"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int64_t released_offsets[3]; + int released_cnt = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-mixed-acks", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 10); + + /* Consumer A: consume all 10 in a single batch, apply mixed ack + * types */ + rkshare = create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + /* Skip partial batches — wait for all 10 in one call */ + if (rcvd < 10) { + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + + for (j = 0; j < rcvd; j++) { + rd_kafka_resp_err_t err; + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + + if (rkm->err) + continue; + + if (consumed < 5) { + /* ACCEPT first 5 */ + err = rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + } else if (consumed < 8) { + /* RELEASE next 3 */ + released_offsets[released_cnt++] = rkm->offset; + err = rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + } else { + /* REJECT last 2 */ + err = rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + } + + TEST_ASSERT(!err, "acknowledge_type failed: %s", + rd_kafka_err2str(err)); + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY( + "Consumer A consumed %d messages " + "(5 ACCEPT, 3 RELEASE, 2 REJECT)\n", + consumed); + TEST_ASSERT(consumed == 10, "Expected 10, got %d", consumed); + TEST_ASSERT(released_cnt == 3, "Expected 3 released, got %d", + released_cnt); + + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT(partitions != NULL, + "Expected per-partition results, got NULL"); + TEST_ASSERT(partitions->cnt == 1, + "Expected exactly 1 partition result, got %d", + partitions->cnt); + TEST_SAY(" %s [%" PRId32 "]: %s\n", partitions->elems[0].topic, + partitions->elems[0].partition, + rd_kafka_err2str(partitions->elems[0].err)); + TEST_ASSERT(!strcmp(partitions->elems[0].topic, topic), + "Expected topic %s, got %s", topic, + partitions->elems[0].topic); + TEST_ASSERT(partitions->elems[0].partition == 0, + "Expected partition 0, got %" PRId32, + partitions->elems[0].partition); + TEST_ASSERT(partitions->elems[0].err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR, got %s", + rd_kafka_err2str(partitions->elems[0].err)); + + rd_kafka_topic_partition_list_destroy(partitions); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Consumer B: should only get the 3 RELEASE'd records */ + rkshare = create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + consumed = 0; + attempts = 0; + while (consumed < 3 && attempts++ < 10) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + int16_t dc; + int k; + rd_bool_t found = rd_false; + + dc = rd_kafka_message_delivery_count(rkm); + TEST_ASSERT( + dc >= 2, + "Consumer B got record at offset %" PRId64 + " with delivery_count=%d, expected >= 2", + rkm->offset, (int)dc); + + /* Verify it is one of the released offsets */ + for (k = 0; k < released_cnt; k++) { + if (rkm->offset == + released_offsets[k]) { + found = rd_true; + break; + } + } + TEST_ASSERT(found, + "Consumer B got offset %" PRId64 + " which was not RELEASE'd", + rkm->offset); + + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumer B consumed %d messages (expected 3 RELEASE'd)\n", + consumed); + TEST_ASSERT(consumed == 3, + "Consumer B got %d records, expected 3 RELEASE'd", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 6: Multiple commit_sync calls. + * + * Consume 50 records, ACCEPT 10 at a time and commit_sync after + * each batch of 10 (5 commit_sync calls total). After each + * commit_sync, immediately call commit_sync again to verify + * a no-op commit (no pending acks) returns NULL/NULL cleanly. + * Consumer B should get 0 records. + * =================================================================== */ +static void do_test_multiple_commit_sync_calls(void) { + const char *topic; + const char *group = "commit-sync-multi-calls"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int commit_cnt = 0; + int acked_since_last_commit = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-multi-calls", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 50); + + rkshare = create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume all 50, commit_sync every 10 records */ + while (consumed < 50 && attempts++ < 60) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + acked_since_last_commit++; + } + + /* commit_sync every 10 records */ + if (acked_since_last_commit == 10) { + partitions = NULL; + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + commit_cnt++; + TEST_ASSERT( + !error, "commit_sync #%d failed: %s", + commit_cnt, + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT( + partitions != NULL, + "commit_sync #%d: expected results, " + "got NULL", + commit_cnt); + TEST_ASSERT(partitions->cnt == 1, + "commit_sync #%d: expected 1 " + "partition, got %d", + commit_cnt, partitions->cnt); + TEST_ASSERT( + !strcmp(partitions->elems[0].topic, topic), + "commit_sync #%d: expected topic " + "%s, got %s", + commit_cnt, topic, + partitions->elems[0].topic); + TEST_ASSERT(partitions->elems[0].partition == 0, + "commit_sync #%d: expected " + "partition 0, got %" PRId32, + commit_cnt, + partitions->elems[0].partition); + TEST_ASSERT( + partitions->elems[0].err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "commit_sync #%d: %s [%" PRId32 "] got %s", + commit_cnt, partitions->elems[0].topic, + partitions->elems[0].partition, + rd_kafka_err2str(partitions->elems[0].err)); + + rd_kafka_topic_partition_list_destroy( + partitions); + TEST_SAY( + "commit_sync #%d OK " + "(consumed %d so far)\n", + commit_cnt, consumed); + acked_since_last_commit = 0; + + /* Immediately call commit_sync again — + * no pending acks, should return + * NULL/NULL */ + partitions = NULL; + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT( + !error, + "back-to-back commit_sync after #%d " + "failed: %s", + commit_cnt, + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT( + partitions == NULL, + "back-to-back commit_sync after #%d: " + "expected NULL partitions, got %d", + commit_cnt, + partitions ? partitions->cnt : -1); + TEST_SAY( + "back-to-back commit_sync after " + "#%d returned NULL as expected\n", + commit_cnt); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumer A consumed %d messages, %d commit_sync calls\n", + consumed, commit_cnt); + TEST_ASSERT(consumed == 50, "Expected 50, got %d", consumed); + TEST_ASSERT(commit_cnt == 5, "Expected 5 commit_sync calls, got %d", + commit_cnt); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Consumer B: should only get the 5 verification records */ + rkshare = create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer B share_poll returned: rcvd=%zu, error=%s\n", rcvd, + error ? rd_kafka_error_string(error) : "none"); + if (error) + rd_kafka_error_destroy(error); + + consumed = 0; + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer B got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer B got %d messages (expected 5)\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 verification records, got %d", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 7: Multi-topic multi-partition commit_sync. + * + * 10 topics with 6 partitions each, 10 messages per partition + * (600 total). Consume in batches, apply mixed ack types + * (ACCEPT ~50%, RELEASE ~30%, REJECT ~20%) and commit_sync + * after each consume_batch call. + * + * RELEASE'd records are redelivered. Continue consuming until + * accepted + rejected == total_msgs (all records settled). + * On delivery_count >= MAX_REDELIVERY_ROUNDS, force ACCEPT. + * Verify commit_sync returns NO_ERROR each time and that + * delivery_count stays within bounds. + * =================================================================== */ +#define MULTI_TP_TOPICS 10 +#define MULTI_TP_PARTITIONS 6 +#define MULTI_TP_MSGS_PER_PARTITION 10 +#define MAX_REDELIVERY_ROUNDS 4 + +static rd_kafka_share_AcknowledgeType_t get_ack_type(int index) { + int r = index % 10; + if (r < 5) + return RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + else if (r < 8) + return RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE; + else + return RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT; +} + +static void do_test_multi_topic_partition(void) { + const char *group = "commit-sync-multi-tp"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + char *topics[MULTI_TP_TOPICS]; + size_t rcvd; + size_t j; + int total_consumed = 0; + int accepted = 0, released = 0, rejected = 0; + int attempts = 0; + int commit_cnt = 0; + int total_msgs = + MULTI_TP_TOPICS * MULTI_TP_PARTITIONS * MULTI_TP_MSGS_PER_PARTITION; + int i, p; + int16_t max_dc_seen = 0; + + SUB_TEST(); + + /* Create topics and produce 10 messages per partition */ + for (i = 0; i < MULTI_TP_TOPICS; i++) { + topics[i] = rd_strdup(test_mk_topic_name("0176-cs-mtp", 1)); + test_create_topic_wait_exists(common_admin, topics[i], + MULTI_TP_PARTITIONS, -1, + 60 * 1000); + for (p = 0; p < MULTI_TP_PARTITIONS; p++) + test_produce_msgs_simple(common_producer, topics[i], p, + MULTI_TP_MSGS_PER_PARTITION); + } + + rkshare = create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, (const char **)topics, MULTI_TP_TOPICS); + + /* Consume until all records are settled + * (accepted + rejected == total_msgs) */ + while (accepted + rejected < total_msgs && attempts++ < 200) { + rd_kafka_topic_partition_list_t *batch_tps; + int batch_cnt = 0; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + /* Track unique topic-partitions in this batch */ + batch_tps = rd_kafka_topic_partition_list_new( + MULTI_TP_TOPICS * MULTI_TP_PARTITIONS); + + for (j = 0; j < rcvd; j++) { + rd_kafka_share_AcknowledgeType_t ack_type; + rd_kafka_resp_err_t err; + int16_t dc; + const char *msg_topic; + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + + if (rkm->err) + continue; + + msg_topic = rd_kafka_topic_name(rkm->rkt); + + /* Add to batch TPs if not already present */ + if (!rd_kafka_topic_partition_list_find( + batch_tps, msg_topic, rkm->partition)) + rd_kafka_topic_partition_list_add( + batch_tps, msg_topic, rkm->partition); + + dc = rd_kafka_message_delivery_count(rkm); + if (dc > max_dc_seen) + max_dc_seen = dc; + + /* On final delivery attempt, force ACCEPT */ + if (dc >= MAX_REDELIVERY_ROUNDS) + ack_type = + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + else + ack_type = get_ack_type(total_consumed); + + err = rd_kafka_share_acknowledge_type(rkshare, rkm, + ack_type); + TEST_ASSERT(!err, "acknowledge_type failed: %s", + rd_kafka_err2str(err)); + + if (ack_type == RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT) + accepted++; + else if (ack_type == + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE) + released++; + else + rejected++; + + total_consumed++; + batch_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + if (batch_cnt == 0) { + rd_kafka_topic_partition_list_destroy(batch_tps); + continue; + } + + /* commit_sync after each batch */ + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + commit_cnt++; + TEST_ASSERT(!error, "commit_sync #%d failed: %s", commit_cnt, + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT(partitions != NULL, + "commit_sync #%d: expected results, got NULL", + commit_cnt); + + /* Verify commit_sync returned exactly the TPs we + * consumed from in this batch */ + TEST_ASSERT(partitions->cnt == batch_tps->cnt, + "commit_sync #%d: expected %d partition(s), got %d", + commit_cnt, batch_tps->cnt, partitions->cnt); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = + &partitions->elems[i]; + TEST_ASSERT(rktpar->err == RD_KAFKA_RESP_ERR_NO_ERROR, + "commit_sync #%d: %s [%" PRId32 "] got %s", + commit_cnt, rktpar->topic, + rktpar->partition, + rd_kafka_err2str(rktpar->err)); + TEST_ASSERT(rd_kafka_topic_partition_list_find( + batch_tps, rktpar->topic, + rktpar->partition) != NULL, + "commit_sync #%d: unexpected partition " + "%s [%" PRId32 "] in results", + commit_cnt, rktpar->topic, + rktpar->partition); + } + + TEST_SAY( + "commit_sync #%d OK (%d in batch, %d total, " + "settled=%d/%d, %d partition results)\n", + commit_cnt, batch_cnt, total_consumed, accepted + rejected, + total_msgs, partitions->cnt); + + rd_kafka_topic_partition_list_destroy(partitions); + rd_kafka_topic_partition_list_destroy(batch_tps); + } + + TEST_SAY( + "Total consumed=%d (original=%d), commit_sync calls=%d, " + "accepted=%d, released=%d, rejected=%d, " + "max delivery_count=%d\n", + total_consumed, total_msgs, commit_cnt, accepted, released, + rejected, (int)max_dc_seen); + + TEST_ASSERT(accepted + rejected == total_msgs, + "Expected accepted(%d) + rejected(%d) == %d", accepted, + rejected, total_msgs); + + /* Verify redelivery happened */ + TEST_ASSERT(max_dc_seen > 1, + "Expected redeliveries (max delivery_count > 1), " + "got max=%d", + (int)max_dc_seen); + + TEST_ASSERT(max_dc_seen <= MAX_REDELIVERY_ROUNDS, + "Max delivery_count=%d exceeds limit=%d", (int)max_dc_seen, + MAX_REDELIVERY_ROUNDS); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + for (i = 0; i < MULTI_TP_TOPICS; i++) + rd_free(topics[i]); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Mock broker infrastructure (same pattern as 0173). + * =================================================================== */ +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(3, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + /* Set auto.offset.reset=earliest so tests that produce + * before consuming see all records. */ + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + return ctx; +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static void +mock_produce_messages(rd_kafka_t *producer, const char *topic, int msgcnt) { + int i; + for (i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "%s-%d", topic, i); + TEST_ASSERT(rd_kafka_producev( + producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Produce failed"); + } + rd_kafka_flush(producer, 5000); +} + +/** + * @brief Create share consumer for mock broker tests. + * + * Unlike create_share_consumer() which uses test_conf_init with real + * broker settings, this uses mock cluster bootstraps. + */ +static rd_kafka_share_t *create_mock_share_consumer(const char *bootstraps, + const char *group_id, + const char *ack_mode) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "share.acknowledgement.mode", ack_mode); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + return rkshare; +} + +static int count_share_ack_requests(rd_kafka_mock_cluster_t *mcluster) { + size_t cnt; + size_t i; + rd_kafka_mock_request_t **requests; + int share_ack_cnt = 0; + + requests = rd_kafka_mock_get_requests(mcluster, &cnt); + + for (i = 0; i < cnt; i++) { + int16_t api_key = rd_kafka_mock_request_api_key(requests[i]); + if (api_key == RD_KAFKAP_ShareAcknowledge) + share_ack_cnt++; + } + + rd_kafka_mock_request_destroy_array(requests, cnt); + return share_ack_cnt; +} + + +/* =================================================================== + * Test 8: Mock — verify commit_sync uses ShareAcknowledge RPC. + * + * Consume 50 records, ACCEPT 10 at a time, commit_sync after + * each batch. Track ShareAcknowledge requests and verify the + * count matches the number of commit_sync calls that returned + * at least 1 partition result. + * =================================================================== */ +static void do_test_mock_uses_share_acknowledge(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + const char *topic = "mock-cs-share-ack"; + const int msgcnt = 50; + int consumed = 0; + int attempts = 0; + int commit_cnt = 0; + int commit_with_partitions = 0; + int acked_since_last_commit = 0; + int share_ack_cnt; + int i; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + TEST_SAY("Mock cluster ready, bootstraps=%s\n", ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + TEST_SAY("Created mock topic [%s] (1 partition)\n", topic); + + mock_produce_messages(ctx.producer, topic, msgcnt); + TEST_SAY("Produced %d messages to [%s]\n", msgcnt, topic); + + rkshare = create_mock_share_consumer(ctx.bootstraps, "sg-mock-cs-ack", + "explicit"); + TEST_SAY( + "Created share consumer (group=sg-mock-cs-ack, " + "ack=explicit)\n"); + + subscribe_consumer(rkshare, &topic, 1); + TEST_SAY("Subscribed to [%s]\n", topic); + + /* Clear and start tracking requests */ + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + TEST_SAY( + "Started mock request tracking; entering consume loop " + "(msgcnt=%d, batch=%d)\n", + msgcnt, CONSUME_ARRAY); + + /* Consume all records, ACCEPT each, commit_sync every 10 */ + while (consumed < msgcnt && attempts++ < 30) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + + TEST_SAY( + "Iter %d: calling share_poll " + "(consumed=%d/%d, acked_since_last=%d)\n", + attempts, consumed, msgcnt, acked_since_last_commit); + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + TEST_SAY( + "Iter %d: share_poll error: %s " + "(rcvd=%zu)\n", + attempts, rd_kafka_error_string(error), + rd_kafka_messages_count(batch)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + TEST_SAY("Iter %d: share_poll returned %zu msg(s)\n", attempts, + rcvd); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_SAY( + " ACK %s [%" PRId32 "] @ offset %" PRId64 + " (delivery_count=%d)\n", + rd_kafka_topic_name(rkm->rkt), + rkm->partition, rkm->offset, + (int)rd_kafka_message_delivery_count(rkm)); + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + acked_since_last_commit++; + } else { + TEST_SAY( + " msg #%zu carries error %s on " + "%s [%" PRId32 "] @ %" PRId64 "\n", + j, rd_kafka_err2name(rkm->err), + rkm->rkt ? rd_kafka_topic_name(rkm->rkt) + : "(no-rkt)", + rkm->partition, rkm->offset); + } + + if (acked_since_last_commit == 10) { + partitions = NULL; + TEST_SAY( + "Calling commit_sync #%d " + "(timeout=30000ms, acked=%d, " + "consumed=%d)\n", + commit_cnt + 1, acked_since_last_commit, + consumed); + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + commit_cnt++; + TEST_SAY( + "commit_sync #%d returned: " + "error=%s, partitions=%s, " + "partition_cnt=%d\n", + commit_cnt, + error ? rd_kafka_error_string(error) + : "NULL", + partitions ? "non-NULL" : "NULL", + partitions ? partitions->cnt : 0); + TEST_ASSERT( + !error, "commit_sync #%d failed: %s", + commit_cnt, + error ? rd_kafka_error_string(error) : ""); + + if (partitions != NULL) { + commit_with_partitions++; + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t + *rktpar = + &partitions->elems[i]; + TEST_SAY( + " commit_sync #%d " + "partition[%d]: " + "%s [%" PRId32 + "] offset=%" PRId64 + " err=%s\n", + commit_cnt, i, + rktpar->topic, + rktpar->partition, + rktpar->offset, + rd_kafka_err2name( + rktpar->err)); + TEST_ASSERT( + rktpar->err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "commit_sync #%d: " + "%s [%" PRId32 "] got %s", + commit_cnt, rktpar->topic, + rktpar->partition, + rd_kafka_err2str( + rktpar->err)); + } + rd_kafka_topic_partition_list_destroy( + partitions); + } + + TEST_SAY( + "commit_sync #%d OK " + "(consumed %d so far)\n", + commit_cnt, consumed); + acked_since_last_commit = 0; + } + } + rd_kafka_messages_destroy(batch); + } + + TEST_SAY( + "Consume loop exited after %d iter(s): " + "consumed=%d/%d, commit_cnt=%d, with_partitions=%d, " + "acked_since_last=%d\n", + attempts, consumed, msgcnt, commit_cnt, commit_with_partitions, + acked_since_last_commit); + TEST_ASSERT(consumed == msgcnt, "Expected %d, got %d", msgcnt, + consumed); + + /* Wait for async ops to complete before counting requests */ + TEST_SAY( + "Sleeping 3s for async ops to drain before counting " + "ShareAcknowledge requests\n"); + rd_sleep(3); + + share_ack_cnt = count_share_ack_requests(ctx.mcluster); + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + TEST_SAY( + "Mock: ShareAcknowledge requests=%d, " + "commit_sync with partitions=%d\n", + share_ack_cnt, commit_with_partitions); + TEST_ASSERT(share_ack_cnt == commit_with_partitions, + "Expected ShareAcknowledge count (%d) == commit_sync with " + "partitions count (%d)", + share_ack_cnt, commit_with_partitions); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + TEST_SAY("Destroying mock test context\n"); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 9: Mock — commit_sync timeout and recovery. + * + * Phase 1: Consume 10 records, ACCEPT all. Set broker RTT to + * 5000ms and call commit_sync with timeout_ms=2000. The call + * should block ~2000ms then return REQUEST_TIMED_OUT (main + * thread commit_sync deadline) or _TIMED_OUT (broker thread + * socket.timeout.ms) per-partition. The broker still processes + * the ack when the delayed response arrives. + * + * TODO KIP-932: Verify and maybe unify the timeout error + * returned by commit_sync. librdkafka can return either + * REQUEST_TIMED_OUT or _TIMED_OUT depending on which timer + * fires first. + * + * Phase 2: Remove RTT, wait 5s for broker to finish processing. + * Second consumer should get 0 records (acks were processed). + * + * Phase 3: Produce 10 more, consume and ACCEPT, commit_sync + * with normal timeout → succeeds. Verifies recovery. + * =================================================================== */ +static void do_test_mock_commit_sync_timeout(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + const char *topic = "mock-cs-timeout"; + const char *t = topic; + const int msgcnt = 10; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed; + int attempts; + int i; + rd_bool_t got_timed_out = rd_false; + rd_ts_t t_start, t_elapsed_ms; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + mock_produce_messages(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, + "sg-mock-cs-timeout", "explicit"); + subscribe_consumer(rkshare, &t, 1); + + /* Phase 1: Consume all 10 records and ACCEPT */ + consumed = 0; + attempts = 0; + while (consumed < msgcnt && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Phase 1: consumed and acknowledged %d messages\n", consumed); + TEST_ASSERT(consumed == msgcnt, "Expected %d, got %d", msgcnt, + consumed); + + /* Inject 5000ms RTT on all brokers */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, -1, 5000); + + /* commit_sync with 2000ms timeout — should block ~2000ms + * then time out */ + t_start = test_clock(); + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 2000, &partitions); + t_elapsed_ms = (test_clock() - t_start) / 1000; + + TEST_SAY("Phase 1: commit_sync returned after %" PRId64 + "ms, error=%s, partitions=%s\n", + t_elapsed_ms, error ? rd_kafka_error_string(error) : "NULL", + partitions ? "non-NULL" : "NULL"); + + /* Verify commit_sync blocked for ~2000ms (allow 1500-3000ms) */ + TEST_ASSERT(t_elapsed_ms >= 1500 && t_elapsed_ms <= 3000, + "Expected commit_sync to block ~2000ms, " + "got %" PRId64 "ms", + t_elapsed_ms); + + /* May return top-level error or per-partition timeout error. + * Accept either: + * - top-level error: main thread commit_sync deadline fired + * first; rd_kafka_error_code(error) can be __TIMED_OUT + * (raw — top-level errors are not run through the per- + * partition translation funnel). + * - per-partition REQUEST_TIMED_OUT: broker-thread reply + * path stamped __TIMED_OUT on batches, translated to + * REQUEST_TIMED_OUT at the app-facing funnel; or the api + * timer cb wrote REQUEST_TIMED_OUT directly. */ + if (error) { + TEST_SAY("Phase 1: top-level error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + got_timed_out = rd_true; + } + + if (partitions) { + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = + &partitions->elems[i]; + TEST_SAY("Phase 1: %s [%" PRId32 "]: %s\n", + rktpar->topic, rktpar->partition, + rd_kafka_err2str(rktpar->err)); + if (rktpar->err == RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT) + got_timed_out = rd_true; + } + rd_kafka_topic_partition_list_destroy(partitions); + } + + TEST_ASSERT(got_timed_out, + "Expected REQUEST_TIMED_OUT or _TIMED_OUT error " + "from commit_sync with short timeout"); + + /* Phase 2: Remove RTT, wait 5s for broker to finish + * processing the delayed response */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, -1, 0); + rd_sleep(5); + + /* Close first consumer */ + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Second consumer: should get 0 records because the broker + * processed the acks despite client-side timeout */ + rkshare = create_mock_share_consumer(ctx.bootstraps, + "sg-mock-cs-timeout", "implicit"); + subscribe_consumer(rkshare, &t, 1); + + consumed = 0; + attempts = 0; + while (attempts++ < 5) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Phase 2: second consumer got %d records (expected 0)\n", + consumed); + TEST_ASSERT(consumed == 0, + "Second consumer got %d records, expected 0 " + "(broker should have processed acks despite " + "client-side timeout)", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Phase 3: Produce more, consume, commit_sync normally — + * verify recovery after timeout */ + mock_produce_messages(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, + "sg-mock-cs-timeout", "explicit"); + subscribe_consumer(rkshare, &t, 1); + + consumed = 0; + attempts = 0; + while (consumed < msgcnt && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Phase 3: consumed %d messages\n", consumed); + TEST_ASSERT(consumed == msgcnt, "Expected %d, got %d", msgcnt, + consumed); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "Phase 3: commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_ASSERT(partitions != NULL, + "Phase 3: expected partition results, got NULL"); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("Phase 3: %s [%" PRId32 "]: %s\n", rktpar->topic, + rktpar->partition, rd_kafka_err2str(rktpar->err)); + TEST_ASSERT(rktpar->err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 3: expected NO_ERROR for %s [%" PRId32 + "], got %s", + rktpar->topic, rktpar->partition, + rd_kafka_err2str(rktpar->err)); + } + + rd_kafka_topic_partition_list_destroy(partitions); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 10: Mixed commit types — commit_async and commit_sync. + * + * Produce 50 records. Consume all, ACCEPT each record + * individually. For the first 10 records call commit_async, + * then call commit_sync for the 11th. Repeat this pattern + * (10 async + 1 sync) until all records are committed. + * + * Verify that commit_sync after async commits completes very + * quickly (acks already sent by async, so sync has little or + * no work to do). Second consumer should get 0 records. + * =================================================================== */ +static void do_test_mixed_commit_types(void) { + const char *topic; + const char *group = "commit-sync-mixed-types"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int async_cnt = 0; + int sync_cnt = 0; + int acked_since_last_sync = 0; + int i; + rd_ts_t t_start, t_elapsed_ms; + rd_ts_t max_sync_elapsed_ms = 0; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-mixed-types", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 50); + + rkshare = create_share_consumer(group, "explicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume all 50 records, ACCEPT each, alternate between + * commit_async (10 times) and commit_sync (1 time) */ + while (consumed < 50 && attempts++ < 60) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + acked_since_last_sync++; + + if (acked_since_last_sync <= 10) { + /* commit_async for first 10 */ + error = rd_kafka_share_commit_async( + rkshare); + TEST_ASSERT( + !error, + "commit_async #%d failed: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + async_cnt++; + } + + if (acked_since_last_sync == 11) { + /* commit_sync on the 11th — + * should be fast since most acks + * were already sent by async */ + partitions = NULL; + t_start = test_clock(); + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + t_elapsed_ms = + (test_clock() - t_start) / 1000; + sync_cnt++; + + if (t_elapsed_ms > max_sync_elapsed_ms) + max_sync_elapsed_ms = + t_elapsed_ms; + + TEST_ASSERT( + !error, + "commit_sync #%d failed: %s", + sync_cnt, + error ? rd_kafka_error_string(error) + : ""); + + if (partitions) { + for (i = 0; i < partitions->cnt; + i++) { + rd_kafka_topic_partition_t + *rktpar = + &partitions + ->elems[i]; + TEST_ASSERT( + rktpar->err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "commit_sync #%d: " + "%s [%" PRId32 + "] got %s", + sync_cnt, + rktpar->topic, + rktpar->partition, + rd_kafka_err2str( + rktpar->err)); + } + rd_kafka_topic_partition_list_destroy( + partitions); + } + + TEST_SAY( + "commit_sync #%d OK " + "in %" PRId64 + "ms (consumed %d, " + "%d async calls)\n", + sync_cnt, t_elapsed_ms, consumed, + async_cnt); + acked_since_last_sync = 0; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Final commit_sync for any remaining acks */ + if (acked_since_last_sync > 0) { + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + sync_cnt++; + TEST_ASSERT(!error, "final commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE(partitions, rd_kafka_topic_partition_list_destroy); + TEST_SAY("Final commit_sync #%d OK\n", sync_cnt); + } + + TEST_SAY( + "Consumed %d, async commits=%d, sync commits=%d, " + "max sync elapsed=%" PRId64 "ms\n", + consumed, async_cnt, sync_cnt, max_sync_elapsed_ms); + TEST_ASSERT(consumed == 50, "Expected 50, got %d", consumed); + + /* Verify commit_sync completed quickly — acks were mostly + * already sent by commit_async, so sync should not wait long. + * Allow up to 1000ms to account for broker round-trip. */ + TEST_ASSERT(max_sync_elapsed_ms < 1000, + "commit_sync took too long (%" PRId64 + "ms), expected < 1000ms since acks were " + "mostly sent by commit_async", + max_sync_elapsed_ms); + + /* Allow time for any in-flight async ack requests to complete + * before closing the consumer. The final commit_async calls + * may have dispatched acks that are still awaiting broker + * response. */ + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Produce 5 verification records */ + test_produce_msgs_simple(common_producer, topic, 0, 5); + + /* Second consumer: should only get the 5 verification records */ + rkshare = create_share_consumer(group, "implicit"); + subscribe_consumer(rkshare, &topic, 1); + + error = rd_kafka_share_poll(rkshare, 15000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_SAY("Consumer B share_poll returned: rcvd=%zu, error=%s\n", rcvd, + error ? rd_kafka_error_string(error) : "none"); + if (error) + rd_kafka_error_destroy(error); + + consumed = 0; + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_ASSERT(rd_kafka_message_delivery_count(rkm) == 1, + "Consumer B got redelivered record at " + "offset %" PRId64 " (delivery_count=%d)", + rkm->offset, + rd_kafka_message_delivery_count(rkm)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Consumer B got %d messages (expected 5)\n", consumed); + TEST_ASSERT(consumed == 5, "Expected 5 verification records, got %d", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test 11: Mock — broker dispatch priority (pending_commit_sync + * dispatched before async_ack_details). + * + * Produce 30 messages. Consume all 30 in one batch. Then: + * - ACCEPT first 10, commit_async → inflight (push entry 1: 2s RTT) + * - ACCEPT next 10, commit_async → cached in async_ack_details + * - ACCEPT last 10, commit_sync(5000) → pending_commit_sync + * + * When inflight completes at ~2s, broker dispatches next request. + * If pending_commit_sync has priority (correct): sync dispatched + * next (push entry 2: 2s RTT), completes at ~4s < 5s → NO_ERROR. + * If async_ack_details has priority (wrong): async dispatched + * next (2s RTT), then sync (2s RTT), completes at ~6s > 5s → + * TIMED_OUT. + * + * Verification: + * 1. commit_sync returns NO_ERROR and completes in ~4s (3500-4500ms) + * 2. Per-partition results show NO_ERROR + * 3. Second consumer gets 0 records (all acks processed) + * =================================================================== */ +static void do_test_mock_broker_dispatch_priority(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + const char *topic = "mock-cs-dispatch-prio"; + const char *t = topic; + const int msgcnt = 30; + rd_kafka_messages_t *batch = NULL; + rd_kafka_messages_t *all_batch = NULL; + size_t rcvd; + size_t j; + int consumed; + int attempts; + int i; + int32_t leader_broker_id = 1; + rd_ts_t t_start, t_elapsed_ms; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, + leader_broker_id); + + mock_produce_messages(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, "sg-mock-cs-prio", + "explicit"); + subscribe_consumer(rkshare, &t, 1); + + /* Consume all 30 records in one batch, keep batch alive for + * later acknowledge calls. */ + consumed = 0; + attempts = 0; + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err && consumed < msgcnt) + consumed++; + } + + if (consumed == 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } else { + all_batch = batch; + batch = NULL; + } + } + + TEST_SAY("Consumed %d/%d messages\n", consumed, msgcnt); + TEST_ASSERT(consumed == msgcnt, "Expected %d, got %d", msgcnt, + consumed); + + /* Push 3 ShareAcknowledge entries with 2000ms RTT each on + * the leader broker. These are consumed in order: + * Entry 1: first commit_async inflight request (~2s) + * Entry 2: pending_commit_sync (if priority correct) (~2s) + * Entry 3: async_ack_details (dispatched last) (~2s) */ + rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, leader_broker_id, RD_KAFKAP_ShareAcknowledge, 3, + RD_KAFKA_RESP_ERR_NO_ERROR, 2000, RD_KAFKA_RESP_ERR_NO_ERROR, 2000, + RD_KAFKA_RESP_ERR_NO_ERROR, 2000); + + /* ACCEPT first 10, commit_async → sends first + * ShareAcknowledge (inflight, push entry 1: 2s RTT) */ + for (i = 0; i < 10; i++) + rd_kafka_share_acknowledge(rkshare, + rd_kafka_messages_get(all_batch, i)); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async #1 failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_SAY("commit_async #1 sent (inflight, 2s RTT)\n"); + + /* Small delay to ensure async request is dispatched to + * broker thread before next commit */ + rd_usleep(200 * 1000, NULL); + + /* ACCEPT next 10, commit_async → broker busy, + * cached in async_ack_details */ + for (i = 10; i < 20; i++) + rd_kafka_share_acknowledge(rkshare, + rd_kafka_messages_get(all_batch, i)); + + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async #2 failed: %s", + error ? rd_kafka_error_string(error) : ""); + TEST_SAY("commit_async #2 sent (cached in async_ack_details)\n"); + + /* ACCEPT last 10, commit_sync(5000ms) → broker still busy, + * stored in pending_commit_sync. + * + * Timeline if sync has dispatch priority (correct): + * t=0s: inflight request sent + * t=2s: inflight completes, sync dispatched (entry 2) + * t=4s: sync completes → NO_ERROR (4s < 5s timeout) + * + * Timeline if async has priority (wrong): + * t=0s: inflight request sent + * t=2s: inflight completes, async dispatched (entry 2) + * t=4s: async completes, sync dispatched (entry 3) + * t=5s: sync timeout fires → TIMED_OUT (5s < 6s) */ + for (i = 20; i < 30; i++) + rd_kafka_share_acknowledge(rkshare, + rd_kafka_messages_get(all_batch, i)); + + TEST_SAY("Calling commit_sync(5000ms)\n"); + t_start = test_clock(); + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 5000, &partitions); + t_elapsed_ms = (test_clock() - t_start) / 1000; + + TEST_SAY("commit_sync returned after %" PRId64 "ms, error=%s\n", + t_elapsed_ms, + error ? rd_kafka_error_string(error) : "NO_ERROR"); + + /* Verify no error */ + if (error) { + TEST_ASSERT(rd_kafka_error_code(error) != + RD_KAFKA_RESP_ERR__TIMED_OUT, + "commit_sync TIMED_OUT after %" PRId64 + "ms — pending_commit_sync was NOT dispatched " + "before async_ack_details. " + "Dispatch priority is broken.", + t_elapsed_ms); + rd_kafka_error_destroy(error); + } else { + TEST_SAY( + "commit_sync returned NO_ERROR — dispatch " + "priority is correct\n"); + } + + /* Verify timing: should complete in ~4s (2s inflight + 2s sync), + * not ~6s (2s inflight + 2s async + 2s sync). */ + TEST_ASSERT(t_elapsed_ms >= 3250 && t_elapsed_ms <= 4500, + "Expected commit_sync to complete in ~4s (3250-4500ms), " + "got %" PRId64 "ms. If >5s, dispatch priority is wrong.", + t_elapsed_ms); + + /* Verify per-partition results show NO_ERROR */ + TEST_ASSERT(partitions != NULL, + "Expected per-partition results, got NULL"); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY(" %s [%" PRId32 "]: %s\n", rktpar->topic, + rktpar->partition, rd_kafka_err2str(rktpar->err)); + TEST_ASSERT(rktpar->err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR for %s [%" PRId32 "], got %s", + rktpar->topic, rktpar->partition, + rd_kafka_err2str(rktpar->err)); + } + + rd_kafka_topic_partition_list_destroy(partitions); + + /* Destroy the held batch (frees all message handles) */ + rd_kafka_messages_destroy(all_batch); + all_batch = NULL; + + /* Wait for remaining async to complete */ + rd_sleep(3); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* Second consumer: should get 0 records since all acks + * were processed successfully */ + rkshare = create_mock_share_consumer(ctx.bootstraps, "sg-mock-cs-prio", + "implicit"); + subscribe_consumer(rkshare, &t, 1); + + consumed = 0; + attempts = 0; + while (attempts++ < 5) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Second consumer got %d records (expected 0)\n", consumed); + TEST_ASSERT(consumed == 0, + "Second consumer got %d records, expected 0 " + "(all acks should have been processed)", + consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* Extended ack callback state with error tracking for sync commit tests */ +typedef struct ack_cb_state_s { + test_ack_cb_state_t base; /* Base state from test.h */ + rd_kafka_resp_err_t errors[64]; /* Track errors per callback */ + int error_cnt; +} ack_cb_state_t; + +static void share_ack_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + ack_cb_state_t *state = (ack_cb_state_t *)opaque; + const rd_kafka_share_partition_offsets_t *entry; + size_t partition_cnt, offsets_in_entry = 0; + + (void)rkshare; + + partition_cnt = rd_kafka_share_partition_offsets_list_count(partitions); + + entry = rd_kafka_share_partition_offsets_list_get(partitions, 0); + if (entry) + offsets_in_entry = + rd_kafka_share_partition_offsets_offsets_cnt(entry); + + TEST_SAY("ACK CALLBACK: err=%s (%d), partitions=%zu, offsets=%zu\n", + rd_kafka_err2name(err), err, partition_cnt, offsets_in_entry); + + test_ack_cb_state_push_err(&state->base, err); + + /* Track this error in our errors array */ + if (state->error_cnt < 64) + state->errors[state->error_cnt++] = err; + + if (entry) + state->base.total_offsets += offsets_in_entry; +} + + + +/* =================================================================== + * Test: commit_sync callback invocation. + * + * Verifies that the runtime acknowledgement callback is invoked after + * commit_sync when using dedicated ShareAcknowledge request. + * =================================================================== */ +static void do_test_commit_sync_callback(void) { + const char *topic; + const char *group = "commit-sync-callback"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + size_t consumed = 0; + int attempts = 0; + test_ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-callback", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, 50); + + rkshare = + test_create_share_consumer_with_cb(group, "explicit", &state, NULL); + const char *grp_conf[] = {"share.auto.offset.reset", "SET", "earliest"}; + test_alter_group_configurations(group, grp_conf, 1); + subscribe_consumer(rkshare, &topic, 1); + + /* Consume and acknowledge messages */ + while (consumed < 20 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("Consumed and acknowledged %zu messages\n", consumed); + TEST_ASSERT(consumed > 0, "Expected to consume some messages"); + + /* Call commit_sync to trigger callback */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "commit_sync failed: %s", + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE(partitions, rd_kafka_topic_partition_list_destroy); + + /* Wait for callback */ + test_wait_for_cb_with_poll(&state, rkshare, 1, 10000); + + TEST_SAY("Callback count=%d, total_offsets=%zu\n", state.callback_cnt, + state.total_offsets); + + TEST_ASSERT(state.callback_cnt == 1, + "Expected callback to be invoked once, got %d", + state.callback_cnt); + TEST_ASSERT(state.total_offsets == consumed, + "Expected %zu offsets in callback, got %zu", consumed, + state.total_offsets); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * commit_sync after subscribed topic deletion surfaces a per-partition + * error. + * + * Produce records, consume them, delete the subscribed topic, then + * call commit_sync. The returned per-partition list should include the + * deleted topic's partition with an error code (typically + * UNKNOWN_TOPIC_OR_PART), not silent success. + * =================================================================== */ +static void do_test_commit_sync_after_topic_deletion(void) { + const char *topic_name; + char *topic_dup; + char *topics_for_delete[1]; + const char *group = "commit-sync-deleted-topic"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + rd_kafka_resp_err_t del_err; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + rd_bool_t topic_err_seen = rd_false; + + SUB_TEST(); + + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) { + TEST_SAY( + "Skipping commit_sync-after-topic-deletion " + "(broker on Windows)\n"); + SUB_TEST_PASS(); + return; + } + + topic_name = test_mk_topic_name("0176-cs-deleted", 1); + topic_dup = rd_strdup(topic_name); + test_create_topic_wait_exists(common_admin, topic_name, 1, -1, + 60 * 1000); + test_produce_msgs_simple(common_producer, topic_name, 0, 5); + + rkshare = create_share_consumer(group, "implicit"); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic_name, 1); + + /* Consume records (so there's something to commit) */ + while (consumed == 0 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed > 0, "Expected to consume records, got 0"); + + /* Delete the subscribed topic */ + TEST_SAY("Deleting topic %s\n", topic_name); + topics_for_delete[0] = topic_dup; + del_err = test_DeleteTopics_simple(common_admin, NULL, + topics_for_delete, 1, NULL); + TEST_ASSERT(del_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "DeleteTopics failed: %s", rd_kafka_err2str(del_err)); + + /* Wait for the topic delete to propagate in the cluster. */ + rd_sleep(3); + + /* commit_sync should return per-partition results — at least one + * partition's err should indicate the deletion. */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_SAY("commit_sync returned: error=%s, partitions=%p\n", + error ? rd_kafka_error_string(error) : "NULL", + (void *)partitions); + + if (error) + rd_kafka_error_destroy(error); + + if (partitions) { + int i; + TEST_SAY("commit_sync returned %d partition entries\n", + partitions->cnt); + for (i = 0; i < partitions->cnt; i++) { + TEST_SAY(" %s [%" PRId32 "]: %s\n", + partitions->elems[i].topic, + partitions->elems[i].partition, + rd_kafka_err2name(partitions->elems[i].err)); + if (partitions->elems[i].err != + RD_KAFKA_RESP_ERR_NO_ERROR) + topic_err_seen = rd_true; + } + rd_kafka_topic_partition_list_destroy(partitions); + } + + TEST_ASSERT(topic_err_seen, + "Expected commit_sync to surface a per-partition error " + "after topic deletion"); + + rd_free(topic_dup); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Chaos: strict per-record ack / commit interleaving. + * + * Pattern over 30 ACCEPTed records: + * Phase A (records 0-14): one commit per ack, alternating + * commit_sync (odd-indexed acks) and + * commit_async (even-indexed acks). + * Phase B (records 15-29): one commit per every 2 acks, + * alternating commit_sync / commit_async + * by pair index. + * Verifies the library survives tight ack/commit alternation and + * that the ack callback fires at least once across the run. + * =================================================================== */ +static void do_test_chaos_111_ack_commit_interleave(void) { + const char *topic; + const char *group = "commit-sync-chaos-111"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int sync_cnt = 0; + int async_cnt = 0; + const int total_msgs = 30; + const int phase_a = 15; /* 1-1 interleave region */ + ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-chaos-111", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, total_msgs); + + rkshare = test_create_share_consumer_with_cb(group, "explicit", + &state.base, share_ack_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed < total_msgs && attempts++ < 60) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd && consumed < total_msgs; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (rkm->err) + continue; + + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + + if (consumed <= phase_a) { + /* Phase A: ack 1 -> commit alternates */ + if (consumed % 2 == 1) { + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT( + !error, + "phase A commit_sync #%d: %s", + sync_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + RD_IF_FREE( + partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } else { + error = rd_kafka_share_commit_async( + rkshare); + TEST_ASSERT( + !error, + "phase A commit_async #%d: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + async_cnt++; + } + } else if (consumed % 2 == 0) { + /* Phase B: commit every 2 records, + * alternating sync/async by pair index */ + int pair_idx = (consumed - phase_a) / 2; + if (pair_idx % 2 == 1) { + error = rd_kafka_share_commit_async( + rkshare); + TEST_ASSERT( + !error, + "phase B commit_async #%d: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + async_cnt++; + } else { + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT( + !error, + "phase B commit_sync #%d: %s", + sync_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + RD_IF_FREE( + partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(consumed == total_msgs, "Expected %d records, consumed %d", + total_msgs, consumed); + + /* Drain any pending async commit callbacks. */ + test_wait_for_cb_with_poll(&state.base, rkshare, sync_cnt + async_cnt, + 10000); + + TEST_SAY( + "chaos 1-1-1: consumed=%d sync=%d async=%d " + "callbacks=%d offsets=%zu last_err=%s\n", + consumed, sync_cnt, async_cnt, state.base.callback_cnt, + state.base.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + TEST_ASSERT(state.base.callback_cnt >= 1, + "Expected at least 1 ack callback, got %d", + state.base.callback_cnt); + TEST_ASSERT( + test_ack_cb_state_first_err(&state.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Ack callback reported err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Chaos: per-record alternating commit_sync / commit_async. + * + * Consume 40 records and, for every single ACCEPTed record, alternate + * between commit_sync and commit_async. Stresses the library's + * bookkeeping under the tightest possible commit cadence. + * =================================================================== */ +static void do_test_chaos_alternating_commits(void) { + const char *topic; + const char *group = "commit-sync-chaos-alt"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int sync_cnt = 0; + int async_cnt = 0; + const int total_msgs = 40; + ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-chaos-alt", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, total_msgs); + + rkshare = test_create_share_consumer_with_cb(group, "explicit", + &state.base, share_ack_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed < total_msgs && attempts++ < 60) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd && consumed < total_msgs; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (rkm->err) + continue; + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + + if (consumed % 2 == 1) { + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT( + !error, "commit_sync #%d: %s", sync_cnt + 1, + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE( + partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } else { + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async #%d: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + async_cnt++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(consumed == total_msgs, "Expected %d records, consumed %d", + total_msgs, consumed); + + test_wait_for_cb_with_poll(&state.base, rkshare, sync_cnt + async_cnt, + 10000); + + TEST_SAY( + "chaos alt: consumed=%d sync=%d async=%d " + "callbacks=%d offsets=%zu last_err=%s\n", + consumed, sync_cnt, async_cnt, state.base.callback_cnt, + state.base.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + TEST_ASSERT( + test_ack_cb_state_first_err(&state.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Ack callback reported err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Chaos: randomized ack types and commit modes (fixed seed). + * + * Seeded PRNG so the sequence is fully reproducible across runs. + * For each ACCEPTed/RELEASEd/REJECTed record we may immediately issue + * a commit (sync or async), with the choices driven by the PRNG. + * Verifies that the lib survives a realistic mixed workload and that + * ACCEPT'd offsets eventually flow through the ack callback without + * errors. + * =================================================================== */ +static void do_test_chaos_random_ack_random_commit(void) { + const char *topic; + const char *group = "commit-sync-chaos-rand"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int processed = 0; + int accepted = 0; + int released = 0; + int rejected = 0; + int attempts = 0; + int sync_cnt = 0; + int async_cnt = 0; + const int produce_cnt = 80; + const int target = 60; /* stop after this many processed */ + ack_cb_state_t state = {0}; + + SUB_TEST(); + + /* Fixed seed: this test must produce the same chaos sequence + * across runs so a failure can be reproduced deterministically. + * Seed is set before any rand() call inside this test; no earlier + * test in main_0176_share_consumer_commit_sync calls rand(). */ + srand(0xC0FFEE); + + topic = test_mk_topic_name("0176-cs-chaos-rand", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, produce_cnt); + + rkshare = test_create_share_consumer_with_cb(group, "explicit", + &state.base, share_ack_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (processed < target && attempts++ < 80) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + /* Drain the entire batch every iteration. Past `target`, + * stop driving chaos but still ACCEPT every record so + * nothing is left ACQUIRED in the lib's inflight map — + * otherwise close/destroy would have to clean it up, + * which the share consumer doesn't currently handle. */ + for (j = 0; j < rcvd; j++) { + rd_kafka_share_AcknowledgeType_t at; + rd_kafka_resp_err_t aerr; + int r; + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + + if (rkm->err) + continue; + + if (processed >= target) { + /* Drain-only path: ACCEPT to release the + * lock without affecting chaos stats. */ + aerr = rd_kafka_share_acknowledge_type( + rkshare, rkm, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(!aerr, "drain ACCEPT failed: %s", + rd_kafka_err2str(aerr)); + continue; + } + + r = rand() % 6; + if (r < 4) { + at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + accepted++; + } else if (r == 4) { + at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE; + released++; + } else { + at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT; + rejected++; + } + + aerr = + rd_kafka_share_acknowledge_type(rkshare, rkm, at); + TEST_ASSERT(!aerr, "acknowledge_type(%d) failed: %s", + (int)at, rd_kafka_err2str(aerr)); + processed++; + + /* ~1 in 3 records: issue a commit. Sync vs async + * decided by another PRNG draw. */ + if (rand() % 3 == 0) { + if (rand() % 2 == 0) { + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT( + !error, "commit_sync #%d: %s", + sync_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + RD_IF_FREE( + partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } else { + error = rd_kafka_share_commit_async( + rkshare); + TEST_ASSERT( + !error, "commit_async #%d: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) + : ""); + async_cnt++; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Final flush. */ + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "final commit_sync: %s", + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE(partitions, rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + + test_wait_for_cb_with_poll(&state.base, rkshare, sync_cnt + async_cnt, + 15000); + + TEST_SAY( + "chaos random: processed=%d accept=%d release=%d reject=%d " + "sync=%d async=%d callbacks=%d offsets=%zu\n", + processed, accepted, released, rejected, sync_cnt, async_cnt, + state.base.callback_cnt, state.base.total_offsets); + + TEST_ASSERT(state.base.callback_cnt >= 1, + "Expected at least 1 ack callback, got %d", + state.base.callback_cnt); + TEST_ASSERT( + test_ack_cb_state_first_err(&state.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Ack callback reported err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Chaos: burst of back-to-back commit_async then a flushing commit_sync. + * + * Consume 50 records. After every ACCEPT, fire 5 commit_async in a row + * followed by 1 commit_sync. Stresses the in-flight async commit queue + * and the path where commit_sync drains a backlog of pending acks. + * =================================================================== */ +static void do_test_chaos_burst_commits(void) { + const char *topic; + const char *group = "commit-sync-chaos-burst"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int sync_cnt = 0; + int async_cnt = 0; + int b; + const int total_msgs = 50; + const int burst_size = 5; + ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-chaos-burst", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, total_msgs); + + rkshare = test_create_share_consumer_with_cb(group, "explicit", + &state.base, share_ack_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed < total_msgs && attempts++ < 60) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd && consumed < total_msgs; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (rkm->err) + continue; + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + + for (b = 0; b < burst_size; b++) { + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT( + !error, "burst commit_async #%d: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) : ""); + async_cnt++; + } + + error = rd_kafka_share_commit_sync(rkshare, 30000, + &partitions); + TEST_ASSERT(!error, "burst commit_sync #%d: %s", + sync_cnt + 1, + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE(partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(consumed == total_msgs, "Expected %d records, consumed %d", + total_msgs, consumed); + + test_wait_for_cb_with_poll(&state.base, rkshare, sync_cnt, 15000); + + TEST_SAY( + "chaos burst: consumed=%d sync=%d async=%d " + "callbacks=%d offsets=%zu\n", + consumed, sync_cnt, async_cnt, state.base.callback_cnt, + state.base.total_offsets); + + TEST_ASSERT( + test_ack_cb_state_first_err(&state.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Ack callback reported err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Chaos: callback-driven receipt accounting under varying batch sizes. + * + * Consume 60 records and commit with progressively varying batch sizes + * (ack 1 -> commit, ack 2 -> commit, ack 3 -> commit, ...) alternating + * between commit_sync and commit_async. After the run, verify the ack + * callback received offset counts that account for all ACCEPTed records. + * =================================================================== */ +static void do_test_chaos_callback_receipt_match(void) { + const char *topic; + const char *group = "commit-sync-chaos-receipts"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + int consumed = 0; + int attempts = 0; + int acked_since_commit = 0; + int batch_target = 1; + int sync_cnt = 0; + int async_cnt = 0; + const int total_msgs = 60; + ack_cb_state_t state = {0}; + + SUB_TEST(); + + topic = test_mk_topic_name("0176-cs-chaos-receipts", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, total_msgs); + + rkshare = test_create_share_consumer_with_cb(group, "explicit", + &state.base, share_ack_cb); + test_share_set_auto_offset_reset(group, "earliest"); + subscribe_consumer(rkshare, &topic, 1); + + while (consumed < total_msgs && attempts++ < 60) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + + for (j = 0; j < rcvd && consumed < total_msgs; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (rkm->err) + continue; + rd_kafka_share_acknowledge(rkshare, rkm); + consumed++; + acked_since_commit++; + + if (acked_since_commit < batch_target) + continue; + + /* Alternate sync (odd batch idx) / async (even). */ + if ((sync_cnt + async_cnt) % 2 == 0) { + error = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT( + !error, "receipts commit_sync #%d: %s", + sync_cnt + 1, + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE( + partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } else { + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT( + !error, "receipts commit_async #%d: %s", + async_cnt + 1, + error ? rd_kafka_error_string(error) : ""); + async_cnt++; + } + acked_since_commit = 0; + batch_target = (batch_target % 5) + 1; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Drain any leftover acked-but-not-committed records. */ + if (acked_since_commit > 0) { + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + TEST_ASSERT(!error, "final commit_sync: %s", + error ? rd_kafka_error_string(error) : ""); + RD_IF_FREE(partitions, rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } + + TEST_ASSERT(consumed == total_msgs, "Expected %d records, consumed %d", + total_msgs, consumed); + + test_wait_for_cb_with_poll(&state.base, rkshare, sync_cnt + async_cnt, + 15000); + + TEST_SAY( + "chaos receipts: consumed=%d sync=%d async=%d " + "callbacks=%d offsets=%zu\n", + consumed, sync_cnt, async_cnt, state.base.callback_cnt, + state.base.total_offsets); + + TEST_ASSERT(state.base.callback_cnt >= 1, + "Expected at least 1 ack callback, got %d", + state.base.callback_cnt); + TEST_ASSERT(state.base.total_offsets >= (size_t)consumed, + "Expected callback to surface at least %d offsets, got %zu", + consumed, state.base.total_offsets); + TEST_ASSERT( + test_ack_cb_state_first_err(&state.base) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Ack callback reported err: %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&state.base))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&state.base); + + SUB_TEST_PASS(); +} + + +int main_0176_share_consumer_commit_sync(int argc, char **argv) { + test_timeout_set(120); + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + /* Real broker tests */ + do_test_basic_implicit_commit_sync(); + do_test_basic_explicit_commit_sync(); + do_test_no_pending_acks(); + do_test_commit_sync_prevents_redelivery(); + do_test_mixed_ack_types(); + do_test_multiple_commit_sync_calls(); + do_test_multi_topic_partition(); + do_test_mixed_commit_types(); + + /* Callback test */ + do_test_commit_sync_callback(); + + /* commit_sync after subscribed topic was deleted */ + do_test_commit_sync_after_topic_deletion(); + + /* Chaos: interleaved ack / commit patterns. */ + do_test_chaos_111_ack_commit_interleave(); + do_test_chaos_alternating_commits(); + do_test_chaos_random_ack_random_commit(); + do_test_chaos_burst_commits(); + do_test_chaos_callback_receipt_match(); + + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} + +int main_0176_share_consumer_commit_sync_local(int argc, char **argv) { + /* Mock broker tests only (no real broker needed) */ + TEST_SKIP_MOCK_CLUSTER(0); + + do_test_mock_uses_share_acknowledge(); + do_test_mock_commit_sync_timeout(); + do_test_mock_broker_dispatch_priority(); + + return 0; +} \ No newline at end of file diff --git a/lib/librdkafka-2.15.0/tests/0177-share_consumer_transactions.c b/lib/librdkafka-2.15.0/tests/0177-share_consumer_transactions.c new file mode 100644 index 00000000000..000f4567fae --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0177-share_consumer_transactions.c @@ -0,0 +1,1815 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2025, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +/** + * @brief Share consumer transaction tests. + * + * Tests that share consumers correctly handle transactional messages + * based on the isolation.level configuration: + * + * - read_committed: Only see committed messages, aborted filtered out + * - read_uncommitted: See ALL messages including aborted ones + * + * Both modes are tested to verify correct behavior. + */ + +#define BATCH_SIZE 1000 + +/* Common handles shared across all tests */ +static rd_kafka_t *common_admin; +static rd_kafka_t *common_regular_producer; + + +/** + * @brief Create a transactional producer. + */ +static rd_kafka_t *create_txn_producer(const char *txn_id) { + rd_kafka_t *rk; + rd_kafka_conf_t *conf; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "transactional.id", txn_id); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + rk = test_create_handle(RD_KAFKA_PRODUCER, conf); + TEST_CALL_ERROR__(rd_kafka_init_transactions(rk, -1)); + + return rk; +} + + +/** + * @brief Configure share group settings. + * + * Sets share.auto.offset.reset=earliest and share.isolation.level + * based on the isolation_level parameter. + * + * Note: share.isolation.level is a BROKER-SIDE share group configuration. + * Transaction filtering happens on the broker, not the client. + */ +static void configure_share_group(const char *group, + const char *isolation_level) { + test_share_set_auto_offset_reset(group, "earliest"); + test_share_set_isolation_level(group, isolation_level); +} + + +/** + * @brief Subscribe share consumer to a topic. + */ +static void subscribe_share_consumer(rd_kafka_share_t *rkshare, + const char *topic) { + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + + err = rd_kafka_share_subscribe(rkshare, subs); + TEST_ASSERT(!err, "Share subscribe failed: %s", rd_kafka_err2str(err)); + + rd_kafka_topic_partition_list_destroy(subs); +} + + +/** + * @brief Consume messages from share consumer. + * + * @returns Number of non-error messages consumed. + */ +static int consume_share_messages(rd_kafka_share_t *rkshare, + int expected_cnt, + int max_attempts, + int poll_timeout_ms) { + rd_kafka_messages_t *batch = NULL; + int consumed = 0; + int attempts = max_attempts; + + while (consumed < expected_cnt && attempts-- > 0) { + size_t rcvd = 0; + size_t i; + rd_kafka_error_t *err; + + err = rd_kafka_share_poll(rkshare, poll_timeout_ms, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, i); + if (!msg->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + return consumed; +} + + +/** + * @brief Poll share consumer expecting no messages (for abort verification). + * + * @returns Number of messages received (should be 0 for aborted txns + * with read_committed). + */ +static int consume_share_no_msgs(rd_kafka_share_t *rkshare, + int poll_attempts, + int poll_timeout_ms) { + rd_kafka_messages_t *batch = NULL; + int consumed = 0; + int attempts = poll_attempts; + + while (attempts-- > 0) { + size_t rcvd = 0; + size_t i; + rd_kafka_error_t *err; + + err = rd_kafka_share_poll(rkshare, poll_timeout_ms, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, i); + if (!msg->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + return consumed; +} + + +/** + * @brief Consume messages and verify exact offsets. + * + * Consumes messages from the share consumer and verifies that: + * 1. The number of messages matches expected_msg_cnt + * 2. The offsets match the expected_offsets array + * 3. None of the forbidden_offsets appear (e.g., control records) + * + * @param rkshare Share consumer instance + * @param expected_msg_cnt Expected number of messages to consume + * @param expected_offsets Array of expected offsets in order + * @param forbidden_offsets Array of offsets that should never appear + * @param forbidden_cnt Number of forbidden offsets + * @param isolation_level Isolation level string (for logging) + */ +static void consume_and_verify_offsets(rd_kafka_share_t *rkshare, + int expected_msg_cnt, + const int64_t *expected_offsets, + const int64_t *forbidden_offsets, + int forbidden_cnt, + const char *isolation_level) { + rd_kafka_messages_t *batch = NULL; + int attempts = 50; + int64_t received_offsets[10]; + int consumed = 0; + int i, j; + + TEST_SAY("[%s] Consuming messages and verifying offsets:\n", + isolation_level); + + /* Consume messages and collect offsets */ + while (consumed < expected_msg_cnt && attempts-- > 0) { + size_t rcvd = 0; + rd_kafka_error_t *err; + + err = rd_kafka_share_poll(rkshare, 3000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < (int)rcvd; i++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, i); + if (!msg->err && consumed < 10) { + received_offsets[consumed] = msg->offset; + TEST_SAY(" Message %d: offset=%" PRId64 "\n", + consumed, msg->offset); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Verify no forbidden offsets were received */ + for (i = 0; i < consumed; i++) { + for (j = 0; j < forbidden_cnt; j++) { + TEST_ASSERT(received_offsets[i] != forbidden_offsets[j], + "[%s] Received forbidden offset %" PRId64 + " at position %d - should be filtered!", + isolation_level, forbidden_offsets[j], i); + } + } + + /* Verify message count */ + TEST_ASSERT(consumed == expected_msg_cnt, + "[%s] Expected %d messages, got %d", isolation_level, + expected_msg_cnt, consumed); + + /* Verify exact offsets */ + for (i = 0; i < consumed; i++) { + TEST_ASSERT(received_offsets[i] == expected_offsets[i], + "[%s] Message %d: expected offset %" PRId64 + ", got %" PRId64, + isolation_level, i, expected_offsets[i], + received_offsets[i]); + } + + TEST_SAY( + "SUCCESS [%s]: All %d offsets verified correctly, " + "forbidden offsets filtered\n", + isolation_level, consumed); +} + + +/** + * @brief Test committed transaction visibility. + * + * Both read_committed and read_uncommitted should see committed messages. + */ +static void do_test_committed_transaction(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-commit-test"; + const int msg_cnt = 100; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int consumed; + + rd_snprintf(topic_suffix, sizeof(topic_suffix), "0177-txn-commit-%s", + isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-commit-%s", + isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Begin transaction */ + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + + /* Produce messages */ + TEST_SAY("Producing %d messages in transaction\n", msg_cnt); + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + + /* Commit transaction */ + TEST_SAY("Committing transaction\n"); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + + /* Consume messages - both modes should receive committed messages */ + TEST_SAY("Consuming messages via share consumer (%s)\n", + isolation_level); + consumed = consume_share_messages(consumer, msg_cnt, 50, 3000); + + TEST_ASSERT(consumed == msg_cnt, + "[%s] Expected %d messages from committed transaction, " + "got %d", + isolation_level, msg_cnt, consumed); + + TEST_SAY( + "SUCCESS [%s]: Share consumer received %d committed " + "messages\n", + isolation_level, consumed); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test aborted transaction visibility. + * + * - read_committed: Should NOT see aborted messages (expect 0) + * - read_uncommitted: Should see ALL messages (expect msg_cnt) + */ +static void do_test_aborted_transaction(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-abort-test"; + const int msg_cnt = 100; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int consumed; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), "0177-txn-abort-%s", + isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-abort-%s", + isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Expected messages depend on isolation level: + * - read_committed: 0 (aborted messages filtered) + * - read_uncommitted: msg_cnt (all messages visible) */ + expected_msgs = is_read_committed ? 0 : msg_cnt; + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Begin transaction */ + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + + /* Produce messages */ + TEST_SAY("Producing %d messages in transaction\n", msg_cnt); + test_curr->ignore_dr_err = rd_true; /* Aborted messages may fail */ + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + + /* Abort transaction */ + TEST_SAY("Aborting transaction\n"); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer, 30000)); + test_curr->ignore_dr_err = rd_false; + + /* Consume messages */ + TEST_SAY("Consuming messages via share consumer (%s), expecting %d\n", + isolation_level, expected_msgs); + + if (is_read_committed) { + consumed = consume_share_no_msgs(consumer, 10, 2000); + } else { + consumed = + consume_share_messages(consumer, expected_msgs, 50, 3000); + } + + TEST_ASSERT(consumed == expected_msgs, + "[%s] Expected %d messages from aborted transaction, " + "got %d", + isolation_level, expected_msgs, consumed); + + TEST_SAY( + "SUCCESS [%s]: Share consumer received %d messages " + "(expected %d)\n", + isolation_level, consumed, expected_msgs); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test mixed transactions (some committed, some aborted). + * + * - read_committed: Only see committed messages + * - read_uncommitted: See ALL messages + */ +static void do_test_mixed_transactions(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-mixed-test"; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int consumed; + int i; + int expected_committed = 0; + int expected_total = 0; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + /* Define transaction scenarios */ + struct { + const char *desc; + rd_bool_t commit; + int msg_cnt; + } txns[] = { + {"Commit 50 msgs", rd_true, 50}, {"Abort 50 msgs", rd_false, 50}, + {"Commit 30 msgs", rd_true, 30}, {"Abort 20 msgs", rd_false, 20}, + {"Commit 40 msgs", rd_true, 40}, + }; + const int txn_cnt = sizeof(txns) / sizeof(txns[0]); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), "0177-txn-mixed-%s", + isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-mixed-%s", + isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Execute transactions */ + for (i = 0; i < txn_cnt; i++) { + TEST_SAY("Transaction %d: %s\n", i + 1, txns[i].desc); + + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + + test_curr->ignore_dr_err = !txns[i].commit; + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, txns[i].msg_cnt, + NULL, 0); + + if (txns[i].commit) { + TEST_CALL_ERROR__( + rd_kafka_commit_transaction(producer, 30000)); + expected_committed += txns[i].msg_cnt; + } else { + TEST_CALL_ERROR__( + rd_kafka_abort_transaction(producer, 30000)); + } + expected_total += txns[i].msg_cnt; + test_curr->ignore_dr_err = rd_false; + } + + /* Expected depends on isolation level */ + expected_msgs = is_read_committed ? expected_committed : expected_total; + + TEST_SAY("Expected messages [%s]: %d (committed=%d, total=%d)\n", + isolation_level, expected_msgs, expected_committed, + expected_total); + + /* Consume messages */ + consumed = consume_share_messages(consumer, expected_msgs, 100, 3000); + + TEST_ASSERT(consumed == expected_msgs, + "[%s] Expected %d messages, got %d", isolation_level, + expected_msgs, consumed); + + TEST_SAY("SUCCESS [%s]: Share consumer received %d messages\n", + isolation_level, consumed); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test control records filtering. + * + * Control records (abort/commit markers) should never be visible to the + * application, regardless of isolation level. + */ +static void do_test_control_records_filtered(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-ctrl-test"; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), "0177-txn-ctrl-%s", + isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-ctrl-%s", isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Transaction 1: Produce and ABORT (1 msg + 1 abort marker) */ + TEST_SAY("Transaction 1: Produce 1 message, then ABORT\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, 1, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer, 30000)); + test_curr->ignore_dr_err = rd_false; + + /* Transaction 2: Produce and COMMIT (5 msgs + 1 commit marker) */ + TEST_SAY("Transaction 2: Produce 5 messages, then COMMIT\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, 5, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + + /* + * Topic now contains: + * - Offset 0: aborted message + * - Offset 1: abort control record + * - Offset 2-6: committed messages + * - Offset 7: commit control record + * + * read_committed: should see offsets 2,3,4,5,6 (committed only) + * read_uncommitted: should see offsets 0,2,3,4,5,6 (aborted + + * committed) Control records (offsets 1 and 7) should NEVER be visible. + */ + expected_msgs = is_read_committed ? 5 : 6; + + /* Consume and verify exact offsets */ + int64_t expected_offsets_committed[] = {2, 3, 4, 5, 6}; + int64_t expected_offsets_uncommitted[] = {0, 2, 3, 4, 5, 6}; + int64_t control_record_offsets[] = {1, 7}; + const int64_t *expected_offsets; + + expected_offsets = is_read_committed ? expected_offsets_committed + : expected_offsets_uncommitted; + + consume_and_verify_offsets(consumer, expected_msgs, expected_offsets, + control_record_offsets, 2, isolation_level); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test multiple partitions with transactions. + */ +static void do_test_multi_partition_transactions(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-multipart-test"; + const int partition_cnt = 3; + const int msgs_per_partition = 30; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int consumed; + int expected_committed; + int expected_total; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), "0177-txn-multipart-%s", + isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-multipart-%s", + isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic with multiple partitions */ + test_create_topic(NULL, topic, partition_cnt, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Transaction 1: Produce to all partitions and COMMIT */ + TEST_SAY("Transaction 1: Produce to all partitions, COMMIT\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, + msgs_per_partition * partition_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + + /* Transaction 2: Produce to all partitions and ABORT */ + TEST_SAY("Transaction 2: Produce to all partitions, ABORT\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, + msgs_per_partition * partition_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer, 30000)); + test_curr->ignore_dr_err = rd_false; + + /* Transaction 3: Produce to all partitions and COMMIT */ + TEST_SAY("Transaction 3: Produce to all partitions, COMMIT\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, + msgs_per_partition * partition_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + + /* Calculate expected messages */ + expected_committed = 2 * msgs_per_partition * partition_cnt; + expected_total = 3 * msgs_per_partition * partition_cnt; + expected_msgs = is_read_committed ? expected_committed : expected_total; + + TEST_SAY("Expected messages [%s]: %d\n", isolation_level, + expected_msgs); + + consumed = consume_share_messages(consumer, expected_msgs, 100, 3000); + + TEST_ASSERT(consumed == expected_msgs, + "[%s] Expected %d messages, got %d", isolation_level, + expected_msgs, consumed); + + TEST_SAY("SUCCESS [%s]: Share consumer received %d messages\n", + isolation_level, consumed); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test interleaved transactions from multiple producers. + */ +static void do_test_interleaved_producers(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const int msg_cnt = 50; + rd_kafka_t *producer1, *producer2; + rd_kafka_share_t *consumer; + int consumed; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), + "0177-txn-interleave-%s", isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-interleave-%s", + isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create two transactional producers */ + producer1 = create_txn_producer("txn-producer-1"); + producer2 = create_txn_producer("txn-producer-2"); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Producer 1: Begin transaction */ + TEST_SAY("Producer 1: Begin transaction\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer1)); + + /* Producer 2: Begin transaction */ + TEST_SAY("Producer 2: Begin transaction\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer2)); + + /* Producer 1: Produce messages */ + TEST_SAY("Producer 1: Produce %d messages\n", msg_cnt); + test_produce_msgs2(producer1, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + + /* Producer 2: Produce messages */ + TEST_SAY("Producer 2: Produce %d messages\n", msg_cnt); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(producer2, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + + /* Producer 1: COMMIT */ + TEST_SAY("Producer 1: COMMIT\n"); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer1, 30000)); + + /* Producer 2: ABORT */ + TEST_SAY("Producer 2: ABORT\n"); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer2, 30000)); + test_curr->ignore_dr_err = rd_false; + + /* Expected: read_committed sees only producer1's msgs, + * read_uncommitted sees both */ + expected_msgs = is_read_committed ? msg_cnt : (2 * msg_cnt); + + consumed = consume_share_messages(consumer, expected_msgs, 50, 3000); + + TEST_ASSERT(consumed == expected_msgs, + "[%s] Expected %d messages, got %d", isolation_level, + expected_msgs, consumed); + + TEST_SAY("SUCCESS [%s]: Share consumer received %d messages\n", + isolation_level, consumed); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer1); + rd_kafka_destroy(producer2); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test non-transactional and transactional messages mixed. + */ +static void do_test_mixed_txn_non_txn(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-mixed-type-test"; + const int non_txn_msgs = 50; + const int txn_commit_msgs = 30; + const int txn_abort_msgs = 20; + rd_kafka_t *txn_producer; + rd_kafka_share_t *consumer; + int consumed; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), + "0177-txn-mixed-type-%s", isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-txn-mixed-type-%s", + isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + txn_producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Produce non-transactional messages */ + TEST_SAY("Producing %d non-transactional messages\n", non_txn_msgs); + test_produce_msgs2(common_regular_producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, non_txn_msgs, NULL, 0); + + /* Transactional: Commit */ + TEST_SAY("Producing %d transactional messages (will commit)\n", + txn_commit_msgs); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(txn_producer)); + test_produce_msgs2(txn_producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, txn_commit_msgs, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(txn_producer, 30000)); + + /* Transactional: Abort */ + TEST_SAY("Producing %d transactional messages (will abort)\n", + txn_abort_msgs); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(txn_producer)); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(txn_producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, txn_abort_msgs, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(txn_producer, 30000)); + test_curr->ignore_dr_err = rd_false; + + /* Expected: + * read_committed: non-txn + committed = 50 + 30 = 80 + * read_uncommitted: all = 50 + 30 + 20 = 100 */ + expected_msgs = is_read_committed + ? (non_txn_msgs + txn_commit_msgs) + : (non_txn_msgs + txn_commit_msgs + txn_abort_msgs); + + TEST_SAY("Expected messages [%s]: %d\n", isolation_level, + expected_msgs); + + consumed = consume_share_messages(consumer, expected_msgs, 100, 3000); + + TEST_ASSERT(consumed == expected_msgs, + "[%s] Expected %d messages, got %d", isolation_level, + expected_msgs, consumed); + + TEST_SAY("SUCCESS [%s]: Share consumer received %d messages\n", + isolation_level, consumed); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(txn_producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test dynamic isolation level change (READ_UNCOMMITTED -> + * READ_COMMITTED). + * + * 1. Start with READ_UNCOMMITTED + * 2. Produce committed txn, consume and acknowledge + * 3. Produce aborted txn, consume and release + * 4. Change to READ_COMMITTED + * 5. Produce more committed/aborted txns + * 6. Verify only committed txns are now returned + */ +static void do_test_dynamic_uncommitted_to_committed(void) { + const char *topic; + const char *group = "share-dynamic-uc-to-c"; + const char *txn_id = "txn-dynamic-uc-c"; + const int msg_cnt = 10; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int consumed; + + SUB_TEST(); + + topic = test_mk_topic_name("0177-dynamic-uc-to-c", 1); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer with READ_UNCOMMITTED */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, "read_uncommitted"); + subscribe_share_consumer(consumer, topic); + + /* Phase 1: Produce and consume committed transaction */ + TEST_SAY("Phase 1: Committed transaction with READ_UNCOMMITTED\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + + consumed = consume_share_messages(consumer, msg_cnt, 50, 3000); + TEST_ASSERT(consumed == msg_cnt, + "Phase 1: Expected %d committed msgs, got %d", msg_cnt, + consumed); + + /* Phase 2: Produce aborted transaction - should be visible in + * READ_UNCOMMITTED */ + TEST_SAY("Phase 2: Aborted transaction with READ_UNCOMMITTED\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer, 30000)); + test_curr->ignore_dr_err = rd_false; + + consumed = consume_share_messages(consumer, msg_cnt, 50, 3000); + TEST_ASSERT(consumed == msg_cnt, + "Phase 2: Expected %d aborted msgs (uncommitted mode), " + "got %d", + msg_cnt, consumed); + + /* Phase 3: Change isolation level to READ_COMMITTED */ + TEST_SAY("Phase 3: Changing isolation level to READ_COMMITTED\n"); + configure_share_group(group, "read_committed"); + + /* Phase 4: Produce aborted transaction - should NOT be visible now */ + TEST_SAY("Phase 4: Aborted transaction with READ_COMMITTED\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer, 30000)); + test_curr->ignore_dr_err = rd_false; + + consumed = consume_share_no_msgs(consumer, 10, 2000); + TEST_ASSERT(consumed == 0, + "Phase 4: Expected 0 aborted msgs (committed mode), got %d", + consumed); + + /* Phase 5: Produce committed transaction - should be visible */ + TEST_SAY("Phase 5: Committed transaction with READ_COMMITTED\n"); + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + + consumed = consume_share_messages(consumer, msg_cnt, 50, 3000); + TEST_ASSERT(consumed == msg_cnt, + "Phase 5: Expected %d committed msgs, got %d", msg_cnt, + consumed); + + TEST_SAY( + "SUCCESS: Dynamic isolation change READ_UNCOMMITTED -> " + "READ_COMMITTED works correctly\n"); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test interval-based abort pattern. + * + * Produce N transactions where every Kth one is aborted. + * Verify READ_COMMITTED skips exactly the aborted ones. + * Verify READ_UNCOMMITTED sees all. + */ +static void do_test_interval_abort_pattern(const char *isolation_level) { + char topic_suffix[64]; + const char *topic; + char group[128]; + const char *txn_id = "txn-interval-abort"; + const int total_txns = 20; + const int abort_interval = 5; /* Every 5th transaction is aborted */ + const int msgs_per_txn = 5; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + int consumed; + int i; + int committed_txns = 0; + int aborted_txns = 0; + int expected_msgs; + rd_bool_t is_read_committed = + !strcmp(isolation_level, "read_committed"); + + rd_snprintf(topic_suffix, sizeof(topic_suffix), "0177-interval-%s", + isolation_level); + topic = test_mk_topic_name(topic_suffix, 1); + rd_snprintf(group, sizeof(group), "share-interval-%s", isolation_level); + + SUB_TEST("isolation.level=%s", isolation_level); + + /* Create topic */ + test_create_topic(NULL, topic, 1, -1); + + /* Create transactional producer */ + producer = create_txn_producer(txn_id); + + /* Create share consumer and set group config */ + consumer = test_create_share_consumer(group, NULL); + configure_share_group(group, isolation_level); + subscribe_share_consumer(consumer, topic); + + /* Produce transactions with interval-based abort pattern */ + TEST_SAY("Producing %d transactions (every %dth is aborted)\n", + total_txns, abort_interval); + + for (i = 1; i <= total_txns; i++) { + rd_bool_t should_abort = (i % abort_interval == 0); + + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + + if (should_abort) + test_curr->ignore_dr_err = rd_true; + + test_produce_msgs2(producer, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msgs_per_txn, NULL, + 0); + + if (should_abort) { + TEST_CALL_ERROR__( + rd_kafka_abort_transaction(producer, 30000)); + test_curr->ignore_dr_err = rd_false; + aborted_txns++; + TEST_SAY(" Transaction %d: ABORTED\n", i); + } else { + TEST_CALL_ERROR__( + rd_kafka_commit_transaction(producer, 30000)); + committed_txns++; + } + } + + TEST_SAY("Produced: %d committed txns, %d aborted txns\n", + committed_txns, aborted_txns); + + /* Calculate expected messages */ + if (is_read_committed) { + expected_msgs = committed_txns * msgs_per_txn; + } else { + expected_msgs = total_txns * msgs_per_txn; + } + + TEST_SAY("[%s] Expected %d messages\n", isolation_level, expected_msgs); + + /* Consume and verify */ + consumed = consume_share_messages(consumer, expected_msgs, 200, 3000); + + TEST_ASSERT(consumed == expected_msgs, + "[%s] Expected %d messages, got %d", isolation_level, + expected_msgs, consumed); + + /* Verify no extra messages */ + { + int extra = consume_share_no_msgs(consumer, 5, 2000); + TEST_ASSERT(extra == 0, "[%s] Unexpected extra messages: %d", + isolation_level, extra); + } + + TEST_SAY( + "SUCCESS [%s]: Interval abort pattern - received exactly %d " + "messages (committed=%d, aborted=%d txns)\n", + isolation_level, consumed, committed_txns, aborted_txns); + + /* Cleanup */ + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Produce exactly one transactional record carrying \p value, then + * commit (\p commit=rd_true) or abort the transaction. + */ +static void produce_msg_txn(rd_kafka_t *producer, + const char *topic, + const char *value, + rd_bool_t commit) { + rd_kafka_resp_err_t err; + + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer)); + + if (!commit) + test_curr->ignore_dr_err = rd_true; + + err = rd_kafka_producev(producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_VALUE((void *)value, strlen(value)), + RD_KAFKA_V_END); + TEST_ASSERT(!err, "producev failed: %s", rd_kafka_err2name(err)); + rd_kafka_flush(producer, 30000); + + if (commit) + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer, 30000)); + else + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer, 30000)); + + test_curr->ignore_dr_err = rd_false; +} + + +/** + * @brief Test: dynamic isolation level change READ_COMMITTED -> + * READ_UNCOMMITTED with RELEASE acknowledgement on a committed + * record. + * + * Phase 1: produce committed Msg1, consume + ACCEPT. + * Phase 2: produce aborted Msg2 (no records visible in read_committed). + * Phase 3: produce committed Msg3, consume + RELEASE. + * Phase 4: produce aborted Msg4. Released Msg3 may reappear; keep RELEASE-ing + * it so the broker can advance past the abort marker. + * Phase 5: alter isolation level to read_uncommitted. + * Phase 6: produce committed Msg5, aborted Msg6, aborted Msg7, committed Msg8. + * Phase 7: consume; the released Msg3 (offset 4) plus the four new messages + * (offsets 8, 10, 12, 14) should all be visible -> 5 unique offsets. + * + * With one record per transaction, the offset layout is: + * 0 Msg1 (commit marker 1) + * 2 Msg2 (abort marker 3) + * 4 Msg3 (commit marker 5) + * 6 Msg4 (abort marker 7) + * 8 Msg5 (commit marker 9) + * 10 Msg6 (abort marker 11) + * 12 Msg7 (abort marker 13) + * 14 Msg8 (commit marker 15) + */ +static void do_test_dynamic_committed_to_uncommitted_with_release(void) { + const char *topic; + const char *group = "share-dynamic-c-to-uc-release"; + const char *txn_id = "txn-dynamic-c-uc-release"; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + rd_kafka_resp_err_t ack_err; + size_t rcvd, j; + int attempts; + int consumed; + rd_bool_t seen[20] = {0}; + const int64_t expected_offsets[] = {4, 8, 10, 12, 14}; + size_t i; + + SUB_TEST(); + + topic = test_mk_topic_name("0177-dyn-c-uc-release", 1); + test_create_topic(NULL, topic, 1, -1); + + producer = create_txn_producer(txn_id); + + consumer = test_create_share_consumer(group, "explicit"); + configure_share_group(group, "read_committed"); + subscribe_share_consumer(consumer, topic); + + /* Phase 1: committed Msg1 -> ACCEPT */ + TEST_SAY("Phase 1: committed Msg1 with READ_COMMITTED\n"); + produce_msg_txn(producer, topic, "Message 1", rd_true); + consumed = 0; + attempts = 0; + while (consumed == 0 && attempts++ < 30) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + TEST_ASSERT(msg->offset == 0, + "Phase 1: expected offset 0, " + "got %" PRId64, + msg->offset); + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 1 ACCEPT failed: %s", + rd_kafka_err2name(ack_err)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed == 1, "Phase 1: expected 1 record, got %d", + consumed); + error = rd_kafka_share_commit_async(consumer); + TEST_ASSERT(!error, "Phase 1 commit_async failed"); + + /* Phase 2: aborted Msg2 - read_committed should hide the record; + * the broker eventually advances past the abort marker. */ + TEST_SAY("Phase 2: aborted Msg2 (read_committed hides it)\n"); + produce_msg_txn(producer, topic, "Message 2", rd_false); + attempts = 10; + while (attempts-- > 0) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 500, &batch); + if (error) + rd_kafka_error_destroy(error); + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + TEST_ASSERT(msg->err || msg->offset != 2, + "Phase 2: aborted Msg2 leaked at " + "offset 2 in read_committed"); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Phase 3: committed Msg3 -> RELEASE */ + TEST_SAY("Phase 3: committed Msg3 -> RELEASE\n"); + produce_msg_txn(producer, topic, "Message 3", rd_true); + consumed = 0; + attempts = 0; + while (consumed == 0 && attempts++ < 30) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + TEST_ASSERT(msg->offset == 4, + "Phase 3: expected offset 4 " + "(Msg3), got %" PRId64, + msg->offset); + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 3 RELEASE failed: %s", + rd_kafka_err2name(ack_err)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed == 1, "Phase 3: expected 1 record, got %d", + consumed); + error = rd_kafka_share_commit_async(consumer); + TEST_ASSERT(!error, "Phase 3 commit_async failed"); + + /* Phase 4: aborted Msg4. Released Msg3 may re-appear; RELEASE it + * again so the next poll can advance. Keep this loop short: each + * RELEASE-and-redeliver bumps Msg3's delivery_count. The default + * share.delivery.count.limit is 5; Phase 3 already used one + * delivery and Phase 7's final ACCEPT will need one, so cap + * Phase 4 at 3 redeliveries to leave headroom and avoid the + * record being archived. */ + TEST_SAY("Phase 4: aborted Msg4; Msg3 may redeliver -> RELEASE\n"); + produce_msg_txn(producer, topic, "Message 4", rd_false); + attempts = 3; + while (attempts-- > 0) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 1000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + /* Should only be Msg3 (offset 4) re-delivered; + * Msg4 at offset 6 is aborted in + * read_committed. + */ + TEST_ASSERT( + msg->offset == 4, + "Phase 4: unexpected offset %" PRId64, + msg->offset); + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 4 RELEASE failed: %s", + rd_kafka_err2name(ack_err)); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + if (rcvd > 0) { + error = rd_kafka_share_commit_async(consumer); + if (error) + rd_kafka_error_destroy(error); + } + } + + /* Phase 5: alter isolation level to read_uncommitted */ + TEST_SAY("Phase 5: alter share.isolation.level to read_uncommitted\n"); + test_share_set_isolation_level(group, "read_uncommitted"); + + /* Phase 6: produce 4 more transactions */ + TEST_SAY("Phase 6: produce Msg5..Msg8\n"); + produce_msg_txn(producer, topic, "Message 5", rd_true); + produce_msg_txn(producer, topic, "Message 6", rd_false); + produce_msg_txn(producer, topic, "Message 7", rd_false); + produce_msg_txn(producer, topic, "Message 8", rd_true); + + /* Phase 7: expect to see 5 unique offsets: 4 (released Msg3), + * 8, 10, 12, 14 (Msg5..Msg8). */ + TEST_SAY("Phase 7: consume - expect offsets {4,8,10,12,14}\n"); + attempts = 0; + while (attempts++ < 60) { + int new_acks = 0; + rd_bool_t all_seen; + rcvd = 0; + error = rd_kafka_share_poll(consumer, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + int64_t off = msg->offset; + if (off >= 0 && off < 20) + seen[off] = rd_true; + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 7 ACCEPT failed: %s", + rd_kafka_err2name(ack_err)); + new_acks++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + if (new_acks > 0) { + error = rd_kafka_share_commit_async(consumer); + if (error) + rd_kafka_error_destroy(error); + } + + all_seen = rd_true; + for (i = 0; + i < sizeof(expected_offsets) / sizeof(expected_offsets[0]); + i++) { + if (!seen[expected_offsets[i]]) { + all_seen = rd_false; + break; + } + } + if (all_seen) + break; + } + + for (i = 0; i < sizeof(expected_offsets) / sizeof(expected_offsets[0]); + i++) { + TEST_ASSERT(seen[expected_offsets[i]], + "Phase 7: expected offset %" PRId64 + " not seen after isolation change", + expected_offsets[i]); + } + + TEST_SAY( + "SUCCESS: dynamic READ_COMMITTED -> READ_UNCOMMITTED with " + "RELEASE\n"); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test: dynamic isolation level change READ_COMMITTED -> + * READ_UNCOMMITTED with REJECT acknowledgement on a committed + * record. + * + * Same setup as the RELEASE variant, except Msg3 is REJECTed instead of + * RELEASEd. After the isolation level switch, Msg3 must NOT be redelivered; + * only the four new messages are expected -> 4 unique offsets. + */ +static void do_test_dynamic_committed_to_uncommitted_with_reject(void) { + const char *topic; + const char *group = "share-dynamic-c-to-uc-reject"; + const char *txn_id = "txn-dynamic-c-uc-reject"; + rd_kafka_t *producer; + rd_kafka_share_t *consumer; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + rd_kafka_resp_err_t ack_err; + size_t rcvd, j; + int attempts; + int consumed; + rd_bool_t seen[20] = {0}; + const int64_t expected_offsets[] = {8, 10, 12, 14}; + size_t i; + + SUB_TEST(); + + topic = test_mk_topic_name("0177-dyn-c-uc-reject", 1); + test_create_topic(NULL, topic, 1, -1); + + producer = create_txn_producer(txn_id); + + consumer = test_create_share_consumer(group, "explicit"); + configure_share_group(group, "read_committed"); + subscribe_share_consumer(consumer, topic); + + /* Phase 1: committed Msg1 -> ACCEPT */ + TEST_SAY("Phase 1: committed Msg1 with READ_COMMITTED\n"); + produce_msg_txn(producer, topic, "Message 1", rd_true); + consumed = 0; + attempts = 0; + while (consumed == 0 && attempts++ < 30) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + TEST_ASSERT(msg->offset == 0, + "Phase 1: expected offset 0, " + "got %" PRId64, + msg->offset); + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 1 ACCEPT failed: %s", + rd_kafka_err2name(ack_err)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed == 1, "Phase 1: expected 1 record, got %d", + consumed); + error = rd_kafka_share_commit_async(consumer); + TEST_ASSERT(!error, "Phase 1 commit_async failed"); + + /* Phase 2: aborted Msg2 */ + TEST_SAY("Phase 2: aborted Msg2 (read_committed hides it)\n"); + produce_msg_txn(producer, topic, "Message 2", rd_false); + attempts = 10; + while (attempts-- > 0) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 500, &batch); + if (error) + rd_kafka_error_destroy(error); + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + TEST_ASSERT(msg->err || msg->offset != 2, + "Phase 2: aborted Msg2 leaked at " + "offset 2 in read_committed"); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Phase 3: committed Msg3 -> REJECT */ + TEST_SAY("Phase 3: committed Msg3 -> REJECT\n"); + produce_msg_txn(producer, topic, "Message 3", rd_true); + consumed = 0; + attempts = 0; + while (consumed == 0 && attempts++ < 30) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + TEST_ASSERT(msg->offset == 4, + "Phase 3: expected offset 4 " + "(Msg3), got %" PRId64, + msg->offset); + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 3 REJECT failed: %s", + rd_kafka_err2name(ack_err)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed == 1, "Phase 3: expected 1 record, got %d", + consumed); + error = rd_kafka_share_commit_async(consumer); + TEST_ASSERT(!error, "Phase 3 commit_async failed"); + + /* Phase 4: aborted Msg4. Msg3 was REJECTed so it should not return. + * Just poll for a while to let the broker process the abort marker. */ + TEST_SAY("Phase 4: aborted Msg4 (Msg3 was REJECTed)\n"); + produce_msg_txn(producer, topic, "Message 4", rd_false); + attempts = 20; + while (attempts-- > 0) { + rcvd = 0; + error = rd_kafka_share_poll(consumer, 500, &batch); + if (error) + rd_kafka_error_destroy(error); + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + TEST_FAIL( + "Phase 4: unexpected record at " + "offset %" PRId64 + " (Msg3 was REJECTed, Msg4 is " + "aborted in read_committed)", + msg->offset); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* Phase 5: alter isolation level to read_uncommitted */ + TEST_SAY("Phase 5: alter share.isolation.level to read_uncommitted\n"); + test_share_set_isolation_level(group, "read_uncommitted"); + + /* Phase 6: produce 4 more transactions */ + TEST_SAY("Phase 6: produce Msg5..Msg8\n"); + produce_msg_txn(producer, topic, "Message 5", rd_true); + produce_msg_txn(producer, topic, "Message 6", rd_false); + produce_msg_txn(producer, topic, "Message 7", rd_false); + produce_msg_txn(producer, topic, "Message 8", rd_true); + + /* Phase 7: expect offsets {8, 10, 12, 14}. Msg3 (offset 4) must NOT + * reappear because it was REJECTed. */ + TEST_SAY("Phase 7: consume - expect offsets {8,10,12,14}, no Msg3\n"); + attempts = 0; + while (attempts++ < 60) { + int new_acks = 0; + rd_bool_t all_seen; + rcvd = 0; + error = rd_kafka_share_poll(consumer, 2000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, j); + if (!msg->err) { + int64_t off = msg->offset; + TEST_ASSERT(off != 4, + "Phase 7: Msg3 (offset 4) " + "should not be redelivered " + "after REJECT"); + if (off >= 0 && off < 20) + seen[off] = rd_true; + ack_err = rd_kafka_share_acknowledge_type( + consumer, msg, + RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(ack_err == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 7 ACCEPT failed: %s", + rd_kafka_err2name(ack_err)); + new_acks++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + if (new_acks > 0) { + error = rd_kafka_share_commit_async(consumer); + if (error) + rd_kafka_error_destroy(error); + } + + all_seen = rd_true; + for (i = 0; + i < sizeof(expected_offsets) / sizeof(expected_offsets[0]); + i++) { + if (!seen[expected_offsets[i]]) { + all_seen = rd_false; + break; + } + } + if (all_seen) + break; + } + + for (i = 0; i < sizeof(expected_offsets) / sizeof(expected_offsets[0]); + i++) { + TEST_ASSERT(seen[expected_offsets[i]], + "Phase 7: expected offset %" PRId64 + " not seen after isolation change", + expected_offsets[i]); + } + TEST_ASSERT(!seen[4], + "Phase 7: Msg3 (offset 4) appeared but it was REJECTed"); + + TEST_SAY( + "SUCCESS: dynamic READ_COMMITTED -> READ_UNCOMMITTED with " + "REJECT\n"); + + test_share_consumer_close(consumer); + test_share_destroy(consumer); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Two share groups, different isolation levels, same topic. + * + * share.isolation.level is a BROKER-SIDE group config — consumers in + * the SAME group cannot have different isolation levels. This test + * exercises two distinct groups consuming the same topic concurrently + * with different isolation levels. + * + * Producer mix on the topic: + * - txn-A committed: 100 records + * - txn-B aborted: 100 records + * - txn-C committed: 100 records + * + * Expected: + * - Group G1 (read_committed): sees 200 records (A+C only) + * - Group G2 (read_uncommitted): sees 300 records (A+B+C) + */ +static void do_test_two_groups_different_isolation_levels(void) { + const char *topic; + const char *grp_committed = "share-txn-twogroup-rc"; + const char *grp_uncommitted = "share-txn-twogroup-ru"; + rd_kafka_t *producer_a; + rd_kafka_t *producer_b; + rd_kafka_t *producer_c; + rd_kafka_share_t *c_committed; + rd_kafka_share_t *c_uncommitted; + const int msg_cnt = 100; + const int expected_rc = 2 * msg_cnt; + const int expected_ru = 3 * msg_cnt; + int consumed_rc; + int consumed_ru; + + SUB_TEST("two groups, different isolation levels"); + + topic = test_mk_topic_name("0177-twogroup-iso", 1); + test_create_topic(NULL, topic, 1, -1); + + producer_a = create_txn_producer("twogroup-iso-A"); + producer_b = create_txn_producer("twogroup-iso-B"); + producer_c = create_txn_producer("twogroup-iso-C"); + + /* Subscribe both consumers BEFORE producing so they're ready to + * consume from the earliest offset. */ + c_committed = test_create_share_consumer(grp_committed, NULL); + configure_share_group(grp_committed, "read_committed"); + subscribe_share_consumer(c_committed, topic); + + c_uncommitted = test_create_share_consumer(grp_uncommitted, NULL); + configure_share_group(grp_uncommitted, "read_uncommitted"); + subscribe_share_consumer(c_uncommitted, topic); + + /* txn-A: committed */ + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer_a)); + test_produce_msgs2(producer_a, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer_a, 30000)); + + /* txn-B: aborted */ + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer_b)); + test_curr->ignore_dr_err = rd_true; + test_produce_msgs2(producer_b, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_abort_transaction(producer_b, 30000)); + test_curr->ignore_dr_err = rd_false; + + /* txn-C: committed */ + TEST_CALL_ERROR__(rd_kafka_begin_transaction(producer_c)); + test_produce_msgs2(producer_c, topic, test_id_generate(), + RD_KAFKA_PARTITION_UA, 0, msg_cnt, NULL, 0); + TEST_CALL_ERROR__(rd_kafka_commit_transaction(producer_c, 30000)); + + /* Both groups consume the same topic concurrently. */ + TEST_SAY("Consuming via read_committed group\n"); + consumed_rc = + consume_share_messages(c_committed, expected_rc, 80, 3000); + TEST_SAY("Consuming via read_uncommitted group\n"); + consumed_ru = + consume_share_messages(c_uncommitted, expected_ru, 80, 3000); + + TEST_SAY( + "Result: read_committed=%d (expected %d), " + "read_uncommitted=%d (expected %d)\n", + consumed_rc, expected_rc, consumed_ru, expected_ru); + + TEST_ASSERT(consumed_rc == expected_rc, + "read_committed group expected %d records, got %d", + expected_rc, consumed_rc); + TEST_ASSERT(consumed_ru == expected_ru, + "read_uncommitted group expected %d records, got %d", + expected_ru, consumed_ru); + + test_share_consumer_close(c_committed); + test_share_destroy(c_committed); + test_share_consumer_close(c_uncommitted); + test_share_destroy(c_uncommitted); + rd_kafka_destroy(producer_a); + rd_kafka_destroy(producer_b); + rd_kafka_destroy(producer_c); + + SUB_TEST_PASS(); +} + + +int main_0177_share_consumer_transactions(int argc, char **argv) { + /* Test both isolation levels */ + const char *isolation_levels[] = {"read_committed", "read_uncommitted"}; + int i; + + test_timeout_set(200); + + /* Create common handles for all tests */ + common_admin = test_create_producer(); + common_regular_producer = test_create_producer(); + + for (i = 0; i < 2; i++) { + const char *level = isolation_levels[i]; + + TEST_SAY("\n"); + TEST_SAY("========================================\n"); + TEST_SAY("Testing with isolation.level=%s\n", level); + TEST_SAY("========================================\n"); + + /* Basic transaction tests */ + do_test_committed_transaction(level); + do_test_aborted_transaction(level); + do_test_mixed_transactions(level); + + /* Control records filtering */ + do_test_control_records_filtered(level); + + /* Multi-partition and multi-producer scenarios */ + do_test_multi_partition_transactions(level); + do_test_interleaved_producers(level); + + /* Mixed workloads */ + do_test_mixed_txn_non_txn(level); + + /* Interval-based abort pattern */ + do_test_interval_abort_pattern(level); + } + + /* Dynamic isolation level change tests (run once, not per-level) */ + TEST_SAY("\n"); + TEST_SAY("========================================\n"); + TEST_SAY("Testing dynamic isolation level changes\n"); + TEST_SAY("========================================\n"); + + do_test_dynamic_uncommitted_to_committed(); + do_test_dynamic_committed_to_uncommitted_with_release(); + do_test_dynamic_committed_to_uncommitted_with_reject(); + + /* Two-groups, different isolation levels on the same topic */ + do_test_two_groups_different_isolation_levels(); + + /* Cleanup common handles */ + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_regular_producer); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0178-share_consumer_close.c b/lib/librdkafka-2.15.0/tests/0178-share_consumer_close.c new file mode 100644 index 00000000000..7317c39340f --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0178-share_consumer_close.c @@ -0,0 +1,2102 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "rdkafka.h" +#include "rdkafka_protocol.h" + +#define MAX_TOPICS 16 +#define MAX_PARTITIONS 32 +#define BATCH_SIZE 10000 +#define MAX_CONSUME_ATTEMPTS 30 + +/** Common producer reused across all real-broker tests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all real-broker tests. */ +static rd_kafka_t *common_admin; + +/** + * @brief Commit mode for test scenarios + */ +typedef enum { + COMMIT_MODE_NONE, /**< No commit before close */ + COMMIT_MODE_ASYNC, /**< Call commitAsync before close */ + COMMIT_MODE_SYNC /**< Call commitSync before close */ +} commit_mode_t; + +/** + * @brief Generic test context for topic and message tracking + */ +typedef struct { + char *topic_names[MAX_TOPICS]; + int topic_cnt; + char *group_id; + int total_msgs_produced; +} test_context_t; + +/** + * @brief Tracked message info for verification + */ +typedef struct { + char *topic; + int32_t partition; + int64_t offset; +} tracked_msg_t; + +/** + * @brief Sink that collects ack receipts from both the share-ack callback + * and synchronous commit_sync results. + * + * Each (topic, partition, offset) is recorded at most once (idempotent add), + * so callback + commit_sync overlaps are harmless. + */ +typedef struct { + tracked_msg_t *receipts; + int receipt_cnt; + int receipt_capacity; + int callback_invocations; /* diagnostic */ + rd_kafka_resp_err_t last_cb_err; + mtx_t lock; +} ack_receipts_t; + +static void ack_receipts_init(ack_receipts_t *r) { + memset(r, 0, sizeof(*r)); + mtx_init(&r->lock, mtx_plain); +} + +static void ack_receipts_destroy(ack_receipts_t *r) { + int i; + for (i = 0; i < r->receipt_cnt; i++) + rd_free(r->receipts[i].topic); + if (r->receipts) + rd_free(r->receipts); + mtx_destroy(&r->lock); +} + +/* Idempotent: dedupes on (topic, partition, offset). */ +static void ack_receipts_add(ack_receipts_t *r, + const char *topic, + int32_t partition, + int64_t offset) { + int i; + mtx_lock(&r->lock); + for (i = 0; i < r->receipt_cnt; i++) { + if (r->receipts[i].partition == partition && + r->receipts[i].offset == offset && + strcmp(r->receipts[i].topic, topic) == 0) { + mtx_unlock(&r->lock); + return; + } + } + if (r->receipt_cnt == r->receipt_capacity) { + int new_cap = + r->receipt_capacity ? r->receipt_capacity * 2 : 16; + r->receipts = + rd_realloc(r->receipts, new_cap * sizeof(*r->receipts)); + r->receipt_capacity = new_cap; + } + r->receipts[r->receipt_cnt].topic = rd_strdup(topic); + r->receipts[r->receipt_cnt].partition = partition; + r->receipts[r->receipt_cnt].offset = offset; + r->receipt_cnt++; + mtx_unlock(&r->lock); +} + +/** + * @brief Share-ack callback that funnels every reported offset into + * the provided ack_receipts_t (passed as opaque). + */ +static void test_0178_ack_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *parts, + rd_kafka_resp_err_t err, + void *opaque) { + ack_receipts_t *r = opaque; + size_t pcnt, p; + + (void)rkshare; + + mtx_lock(&r->lock); + r->callback_invocations++; + r->last_cb_err = err; + mtx_unlock(&r->lock); + + pcnt = rd_kafka_share_partition_offsets_list_count(parts); + TEST_SAY("ack_cb: invocation=%d err=%s partitions=%zu\n", + r->callback_invocations, rd_kafka_err2name(err), pcnt); + + if (err != RD_KAFKA_RESP_ERR_NO_ERROR) + return; + + for (p = 0; p < pcnt; p++) { + const rd_kafka_share_partition_offsets_t *entry = + rd_kafka_share_partition_offsets_list_get(parts, p); + const rd_kafka_topic_partition_t *tp; + const int64_t *offsets; + int ocnt, o; + + if (!entry) + continue; + + tp = rd_kafka_share_partition_offsets_partition(entry); + offsets = rd_kafka_share_partition_offsets_offsets(entry); + ocnt = rd_kafka_share_partition_offsets_offsets_cnt(entry); + + for (o = 0; o < ocnt; o++) + ack_receipts_add(r, tp->topic, tp->partition, + offsets[o]); + } +} + +/** + * @brief Build a share consumer with our ack callback wired in. + * + * Mirrors what test_create_share_consumer_with_cb does, but uses + * @p receipts as the opaque so test_0178_ack_cb can record into it. + */ +static rd_kafka_share_t * +create_share_consumer_with_receipts(const char *group_id, + const char *ack_mode, + ack_receipts_t *receipts) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 60); + rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "share.acknowledgement.mode", ack_mode, errstr, + sizeof(errstr)); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + + rd_kafka_error_t *error = rd_kafka_share_set_acknowledgement_commit_cb( + rkshare, test_0178_ack_cb, receipts); + TEST_ASSERT(error == NULL, + "Failed to set acknowledgement commit callback: %s", + rd_kafka_error_string(error)); + return rkshare; +} + +/** + * @brief Assert that @p r contains a receipt for every entry in + * @p expected[0..expected_cnt). Extra receipts trigger a warning. + */ +static void verify_receipts_match(ack_receipts_t *r, + const tracked_msg_t *expected, + int expected_cnt, + const char *context_label) { + rd_bool_t *seen = + expected_cnt > 0 ? rd_calloc(expected_cnt, sizeof(*seen)) : NULL; + int matched = 0; + int i, j; + + mtx_lock(&r->lock); + + for (i = 0; i < r->receipt_cnt; i++) { + rd_bool_t expected_hit = rd_false; + for (j = 0; j < expected_cnt; j++) { + if (seen[j]) + continue; + if (r->receipts[i].partition == expected[j].partition && + r->receipts[i].offset == expected[j].offset && + strcmp(r->receipts[i].topic, expected[j].topic) == + 0) { + seen[j] = rd_true; + expected_hit = rd_true; + matched++; + break; + } + } + if (!expected_hit) { + TEST_WARN( + "ack receipts: unexpected receipt for %s [%d] " + "@ offset %" PRId64 " (context: %s)\n", + r->receipts[i].topic, r->receipts[i].partition, + r->receipts[i].offset, context_label); + } + } + + for (j = 0; j < expected_cnt; j++) { + if (!seen[j]) { + mtx_unlock(&r->lock); + TEST_FAIL( + "ack receipts: missing receipt for %s [%d] " + "@ offset %" PRId64 + " (context: %s, " + "matched=%d/%d, total receipts=%d, cb invocations=" + "%d)", + expected[j].topic, expected[j].partition, + expected[j].offset, context_label, matched, + expected_cnt, r->receipt_cnt, + r->callback_invocations); + } + } + + TEST_SAY( + "ack receipts: matched %d/%d expected (context: %s, total " + "receipts=%d, cb invocations=%d)\n", + matched, expected_cnt, context_label, r->receipt_cnt, + r->callback_invocations); + + mtx_unlock(&r->lock); + if (seen) + rd_free(seen); +} + +/** + * @brief Get string representation of commit mode + */ +static const char *commit_mode_str(commit_mode_t mode) { + switch (mode) { + case COMMIT_MODE_NONE: + return "no-commit"; + case COMMIT_MODE_ASYNC: + return "commit-async"; + case COMMIT_MODE_SYNC: + return "commit-sync"; + default: + return "unknown"; + } +} + +/** + * @brief Setup topics and produce messages (modular helper) + * + * @param ctx Test context to populate + * @param topic_cnt Number of topics to create + * @param partitions Array of partition counts per topic + * @param msgs_per_partition Messages to produce per partition + * + * @returns Total number of messages produced + */ +static int setup_topics_and_produce(test_context_t *ctx, + int topic_cnt, + const int *partitions, + int msgs_per_partition) { + int t, p; + int total_msgs = 0; + + ctx->topic_cnt = topic_cnt; + + for (t = 0; t < topic_cnt; t++) { + ctx->topic_names[t] = + rd_strdup(test_mk_topic_name("0178-close-test", 1)); + + TEST_SAY("Creating topic %s with %d partition(s)\n", + ctx->topic_names[t], partitions[t]); + + test_create_topic_wait_exists(common_admin, ctx->topic_names[t], + partitions[t], -1, 60 * 1000); + + for (p = 0; p < partitions[t]; p++) { + test_produce_msgs_simple(common_producer, + ctx->topic_names[t], p, + msgs_per_partition); + total_msgs += msgs_per_partition; + } + + TEST_SAY( + "Topic %s: produced %d messages (%d partitions * %d" + "msgs)\n", + ctx->topic_names[t], partitions[t] * msgs_per_partition, + partitions[t], msgs_per_partition); + } + + ctx->total_msgs_produced = total_msgs; + TEST_SAY("Total messages produced: %d\n", total_msgs); + return total_msgs; +} + +/** + * @brief Subscribe consumer to topics + * + * @param rkshare Consumer handle + * @param topic_names Array of topic names to subscribe to + * @param topic_cnt Number of topics + */ +static void subscribe_consumer(rd_kafka_share_t *rkshare, + char **topic_names, + int topic_cnt) { + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + int t; + + /* Build subscription list */ + subs = rd_kafka_topic_partition_list_new(topic_cnt); + for (t = 0; t < topic_cnt; t++) { + rd_kafka_topic_partition_list_add(subs, topic_names[t], + RD_KAFKA_PARTITION_UA); + } + + /* Subscribe */ + err = rd_kafka_share_subscribe(rkshare, subs); + if (err) { + TEST_FAIL("Subscribe failed: %s", rd_kafka_err2str(err)); + } + + rd_kafka_topic_partition_list_destroy(subs); +} + +/** + * @brief Consume messages, track them, and optionally ACCEPT-acknowledge. + * + * In ack mode (do_acknowledge=true), messages are acknowledged with ACCEPT; + * any remaining messages in a batch beyond target_count are RELEASE'd so they + * can be redelivered. In no-ack mode (do_acknowledge=false), messages are + * tracked and destroyed without acknowledging — close() should release them. + * + * @param rkshare Consumer handle + * @param consumer_name Name for logging (e.g., "C1", "C2") + * @param target_count Number of messages to track (and ACCEPT in ack mode) + * @param do_acknowledge If true, ACCEPT each tracked message and RELEASE the + * rest of the batch; if false, track-and-destroy only. + * @param tracked_msgs If non-NULL, track messages here + * @param tracked_cnt If non-NULL, store count of tracked messages here + * + * @returns Number of messages tracked + */ +static int consume_and_track(rd_kafka_share_t *rkshare, + const char *consumer_name, + int target_count, + rd_bool_t do_acknowledge, + tracked_msg_t *tracked_msgs, + int *tracked_cnt) { + rd_kafka_messages_t *batch = NULL; + int attempt; + rd_kafka_error_t *error; + int tracked = 0; + + TEST_SAY("%s: Consuming and %s %d messages (max %d attempts)...\n", + consumer_name, + do_acknowledge ? "acknowledging" : "tracking (no ack)", + target_count, MAX_CONSUME_ATTEMPTS); + + for (attempt = 0; + attempt < MAX_CONSUME_ATTEMPTS && tracked < target_count; + attempt++) { + size_t rcvd_msgs = 0; + int i; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + + if (error) { + TEST_SAY("%s: Attempt %d/%d: error: %s\n", + consumer_name, attempt + 1, + MAX_CONSUME_ATTEMPTS, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + continue; + } + + rcvd_msgs = rd_kafka_messages_count(batch); + + if (rcvd_msgs == 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + + TEST_SAY("%s: Attempt %d/%d: Received %d messages\n", + consumer_name, attempt + 1, MAX_CONSUME_ATTEMPTS, + (int)rcvd_msgs); + + for (i = 0; i < (int)rcvd_msgs && tracked < target_count; i++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, i); + + if (rkm->err) { + TEST_SAY( + "%s: Skipping message %d with error: %s\n", + consumer_name, i, + rd_kafka_message_errstr(rkm)); + continue; + } + + if (tracked_msgs && tracked < BATCH_SIZE) { + tracked_msgs[tracked].topic = + rd_strdup(rd_kafka_topic_name(rkm->rkt)); + tracked_msgs[tracked].partition = + rkm->partition; + tracked_msgs[tracked].offset = rkm->offset; + } + + TEST_SAY("%s: %s message %d: %s [%d] @ offset %" PRId64 + "\n", + consumer_name, + do_acknowledge ? "Acking" : "Tracking", + tracked, rd_kafka_topic_name(rkm->rkt), + rkm->partition, rkm->offset); + + if (do_acknowledge) + rd_kafka_share_acknowledge(rkshare, rkm); + tracked++; + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + + if (tracked >= target_count) { + TEST_SAY("%s: Reached target of %d tracked messages\n", + consumer_name, target_count); + break; + } + } + + TEST_SAY("%s: Finished - tracked %d messages\n", consumer_name, + tracked); + + if (tracked < target_count) { + TEST_FAIL("%s: Expected to track %d messages, got %d", + consumer_name, target_count, tracked); + } + + if (tracked_cnt) + *tracked_cnt = tracked; + + return tracked; +} + +/** + * @brief Execute commit based on mode (modular helper) + * + * @param rkshare Consumer handle + * @param consumer_name Name for logging + * @param mode Commit mode to execute + */ +static void perform_commit(rd_kafka_share_t *rkshare, + const char *consumer_name, + commit_mode_t mode, + ack_receipts_t *receipts) { + rd_kafka_error_t *error; + + switch (mode) { + case COMMIT_MODE_NONE: + TEST_SAY("%s: Skipping commit (mode: %s)\n", consumer_name, + commit_mode_str(mode)); + break; + + case COMMIT_MODE_ASYNC: + TEST_SAY("%s: Calling commitAsync\n", consumer_name); + error = rd_kafka_share_commit_async(rkshare); + if (error) { + TEST_FAIL("%s: commitAsync failed: %s", consumer_name, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } + /* Give async commit some time to process */ + rd_sleep(1); + break; + + case COMMIT_MODE_SYNC: { + rd_kafka_topic_partition_list_t *partitions = NULL; + int p; + TEST_SAY("%s: Calling commitSync\n", consumer_name); + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + if (error) { + TEST_FAIL("%s: commitSync failed: %s", consumer_name, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } + /* Honour synchronous per-partition results. IN_PROGRESS means + * the result will arrive via the ack callback later, so skip + * — the callback path will fill the receipt. */ + if (receipts && partitions) { + for (p = 0; p < partitions->cnt; p++) { + rd_kafka_topic_partition_t *tp = + &partitions->elems[p]; + if (tp->err == RD_KAFKA_RESP_ERR__IN_PROGRESS) + continue; + if (tp->err != RD_KAFKA_RESP_ERR_NO_ERROR) + TEST_FAIL( + "%s: commitSync per-partition " + "error for %s [%d]: %s", + consumer_name, tp->topic, + tp->partition, + rd_kafka_err2str(tp->err)); + ack_receipts_add(receipts, tp->topic, + tp->partition, tp->offset); + } + } + if (partitions) + rd_kafka_topic_partition_list_destroy(partitions); + TEST_SAY("%s: commitSync completed successfully\n", + consumer_name); + break; + } + + default: + TEST_FAIL("Unknown commit mode: %d", mode); + } +} + +/** + * @brief Verify whether tracked messages are (or are not) redelivered. + * + * Consumes up to @p max_total_msgs messages and compares each against the + * @p tracked_msgs list. Behavior depends on @p expect_redelivery: + * - false: fails if any tracked message is received (post-ack scenario); + * also asserts the total received count equals @p max_total_msgs. + * - true: fails unless every tracked message is received (post-release + * scenario); stops early once all tracked messages have been + * matched. + * + * @param rkshare Consumer handle to check + * @param consumer_name Name for logging + * @param max_total_msgs Number of messages to consume (also the exact-count + * assertion in the not-redelivered case) + * @param tracked_msgs Array of tracked messages to check against + * @param tracked_cnt Number of tracked messages + * @param expect_redelivery true = expect all tracked redelivered; false = + * expect none redelivered + * @param context_label Free-form label included in failure messages + * (e.g., commit mode name) + */ +static void verify_tracked_messages(rd_kafka_share_t *rkshare, + const char *consumer_name, + int max_total_msgs, + const tracked_msg_t *tracked_msgs, + int tracked_cnt, + rd_bool_t expect_redelivery, + const char *context_label) { + rd_kafka_messages_t *batch = NULL; + size_t total_rcvd = 0; + int attempt; + rd_kafka_error_t *error; + int matched_count = 0; + rd_bool_t *tracked_seen = + tracked_cnt > 0 ? rd_calloc(tracked_cnt, sizeof(*tracked_seen)) + : NULL; + + TEST_SAY( + "%s: Consuming for verification (up to %d messages, max %d " + "attempts, expect_redelivery=%s)...\n", + consumer_name, max_total_msgs, MAX_CONSUME_ATTEMPTS, + expect_redelivery ? "true" : "false"); + + for (attempt = 0; attempt < MAX_CONSUME_ATTEMPTS && + (int)total_rcvd < max_total_msgs && + (!expect_redelivery || matched_count < tracked_cnt); + attempt++) { + size_t rcvd_msgs = 0; + size_t i; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + + if (error) { + TEST_SAY("%s: Attempt %d/%d: error: %s\n", + consumer_name, attempt + 1, + MAX_CONSUME_ATTEMPTS, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + continue; + } + + rcvd_msgs = rd_kafka_messages_count(batch); + + if (rcvd_msgs == 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + + TEST_SAY( + "%s: Attempt %d/%d: Received %d messages (total: %d)\n", + consumer_name, attempt + 1, MAX_CONSUME_ATTEMPTS, + (int)rcvd_msgs, (int)(total_rcvd + rcvd_msgs)); + + /* Match each new message against the tracked list */ + for (i = 0; i < rcvd_msgs; i++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, i); + const char *topic; + int32_t partition; + int64_t offset; + int j; + + if (!rkm || rkm->err) + continue; + + topic = rd_kafka_topic_name(rkm->rkt); + partition = rkm->partition; + offset = rkm->offset; + + for (j = 0; j < tracked_cnt; j++) { + if (tracked_seen && tracked_seen[j]) + continue; + if (strcmp(topic, tracked_msgs[j].topic) == 0 && + partition == tracked_msgs[j].partition && + offset == tracked_msgs[j].offset) { + if (tracked_seen) + tracked_seen[j] = rd_true; + matched_count++; + if (expect_redelivery) { + TEST_SAY( + "%s: Matched tracked " + "message %d/%d: %s [%d] " + "@ offset %" PRId64 "\n", + consumer_name, + matched_count, tracked_cnt, + topic, partition, offset); + } else { + TEST_WARN( + "%s: Received previously " + "acknowledged message: " + "%s [%d] @ offset %" PRId64 + "\n", + consumer_name, topic, + partition, offset); + } + break; + } + } + } + + total_rcvd += rcvd_msgs; + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("%s: Received %d messages, matched %d/%d tracked\n", + consumer_name, (int)total_rcvd, matched_count, tracked_cnt); + + if (expect_redelivery) { + int j; + if (matched_count != tracked_cnt) { + for (j = 0; j < tracked_cnt; j++) { + if (!tracked_seen[j]) { + TEST_FAIL( + "%s: Did not receive tracked " + "message: %s [%d] @ offset " + "%" PRId64 " (context: %s)", + consumer_name, + tracked_msgs[j].topic, + tracked_msgs[j].partition, + tracked_msgs[j].offset, + context_label); + } + } + } + TEST_SAY( + "%s: Verification passed - all %d tracked messages " + "redelivered\n", + consumer_name, matched_count); + } else { + if ((int)total_rcvd != max_total_msgs) { + TEST_FAIL("%s: Expected to receive %d messages, got %d", + consumer_name, max_total_msgs, + (int)total_rcvd); + } + if (matched_count > 0) { + TEST_FAIL( + "%s received %d messages that were acknowledged " + "(context: %s)", + consumer_name, matched_count, context_label); + } + TEST_SAY( + "%s: Verification passed - no acknowledged messages " + "were redelivered\n", + consumer_name); + } + + if (tracked_seen) + rd_free(tracked_seen); +} + +/** + * @brief Free tracked messages (modular helper) + * + * @param tracked_msgs Array of tracked messages + * @param tracked_cnt Number of tracked messages + */ +static void free_tracked_messages(tracked_msg_t *tracked_msgs, + int tracked_cnt) { + for (int i = 0; i < tracked_cnt; i++) { + if (tracked_msgs[i].topic) + rd_free(tracked_msgs[i].topic); + } +} + +/** + * @brief Helper: assert the given rd_kafka_error_t* signals closed/closing. + */ +static void assert_state_error(rd_kafka_error_t *error, const char *api_name) { + TEST_ASSERT(error != NULL, "%s: expected error, got NULL", api_name); + TEST_ASSERT(rd_kafka_error_code(error) == RD_KAFKA_RESP_ERR__STATE, + "%s: expected __STATE, got %s", api_name, + rd_kafka_err2name(rd_kafka_error_code(error))); + TEST_ASSERT(strstr(rd_kafka_error_string(error), "closed") != NULL, + "%s: expected error string to contain 'closed', got '%s'", + api_name, rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); +} + +/** + * @brief Invoke every share-consumer API and assert each returns a + * closed state error. + * + * @param consumer Share consumer (already closed or closing). + * @param topic Topic name (for subscribe/acknowledge_offset). + */ +static void verify_all_apis_return_error(rd_kafka_share_t *consumer, + const char *topic) { + rd_kafka_error_t *error; + rd_kafka_resp_err_t err; + rd_kafka_messages_t *batch = NULL; + rd_kafka_topic_partition_list_t *subs, *sub_result = NULL; + rd_kafka_topic_partition_list_t *commit_results = NULL; + rd_kafka_queue_t *queue = NULL; + + /* 1. share_poll */ + error = rd_kafka_share_poll(consumer, 100, &batch); + assert_state_error(error, "share_poll"); + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + /* 2. commit_async */ + error = rd_kafka_share_commit_async(consumer); + assert_state_error(error, "commit_async"); + + /* 3. commit_sync */ + error = rd_kafka_share_commit_sync(consumer, 1000, &commit_results); + assert_state_error(error, "commit_sync"); + TEST_ASSERT(commit_results == NULL, + "commit_sync: expected no partitions on error"); + + /* 4. acknowledge_offset */ + err = rd_kafka_share_acknowledge_offset( + consumer, topic, 0, 0, RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "acknowledge_offset: expected __STATE, got %s", + rd_kafka_err2name(err)); + + /* 5. subscribe */ + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(consumer, subs); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "subscribe: expected __STATE, got %s", + rd_kafka_err2name(err)); + rd_kafka_topic_partition_list_destroy(subs); + + /* 6. unsubscribe */ + err = rd_kafka_share_unsubscribe(consumer); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "unsubscribe: expected __STATE, got %s", + rd_kafka_err2name(err)); + + /* 7. subscription */ + err = rd_kafka_share_subscription(consumer, &sub_result); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR__STATE, + "subscription: expected __STATE, got %s", + rd_kafka_err2name(err)); + TEST_ASSERT(sub_result == NULL, "subscription: expected NULL result"); + + /* 8. close */ + error = rd_kafka_share_consumer_close(consumer); + assert_state_error(error, "close"); + + /* 9. async close */ + queue = rd_kafka_queue_new(test_share_consumer_get_rk(consumer)); + error = rd_kafka_share_consumer_close_queue(consumer, queue); + rd_kafka_queue_destroy(queue); + assert_state_error(error, "close_queue"); +} + +/** + * @brief Common setup for 3-broker mock close-tests. + * + * Creates a 3-broker mock cluster, a topic with one partition per broker, + * produces 10 messages to each partition, then creates a share consumer + * (with optional socket.timeout.ms), subscribes, and consumes all + * @p partition_cnt * @p msgs_per_partition messages so the share session + * is established with every partition leader. + * + * @param test_name Used to derive unique topic and group names. + * @param socket_timeout_ms Pass 0 to leave socket.timeout.ms at default, + * or a positive value to override it. + * @param out_mcluster Output: created cluster (caller destroys). + * @param out_consumer Output: subscribed share consumer (caller destroys). + */ +static void setup_3broker_share_consumer(const char *test_name, + int socket_timeout_ms, + rd_kafka_mock_cluster_t **out_mcluster, + rd_kafka_share_t **out_consumer) { + const int partition_cnt = 3; + const int msgs_per_partition = 10; + const int total_msgs = partition_cnt * msgs_per_partition; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + char topic[64], group[64], errstr[512]; + int p, consumed = 0, attempts = 0; + size_t i, rcvd; + + rd_snprintf(topic, sizeof(topic), "mock-%s", test_name); + rd_snprintf(group, sizeof(group), "sg-%s", test_name); + + mcluster = test_mock_cluster_new(3, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, partition_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Produce msgs_per_partition msgs to each partition via test helper. */ + for (p = 0; p < partition_cnt; p++) { + test_produce_msgs_easy_v(topic, 0, p, 0, msgs_per_partition, 16, + "bootstrap.servers", bootstraps, NULL); + } + + /* Consumer */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + if (socket_timeout_ms > 0) { + rd_snprintf(errstr, sizeof(errstr), "%d", socket_timeout_ms); + test_conf_set(conf, "socket.timeout.ms", errstr); + } + + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create consumer: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, subs), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume all messages so share session is established with every + * partition leader (i.e. all 3 brokers). */ + TEST_SAY("Consuming %d messages across %d partitions...\n", total_msgs, + partition_cnt); + while (consumed < total_msgs && attempts++ < 30) { + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, i); + if (rkm && !rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(consumed == total_msgs, "Expected %d messages, got %d", + total_msgs, consumed); + TEST_SAY("Consumed all %d messages\n", consumed); + + *out_mcluster = mcluster; + *out_consumer = consumer; +} + +/** is_fatal_cb hook scoped to test_close_with_broker_down: ignores + * the transport error and cascading ALL_BROKERS_DOWN that result from + * taking the mock broker down mid-close. */ +static int test_close_with_broker_down_is_fatal_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + const char *reason) { + if (err == RD_KAFKA_RESP_ERR__TRANSPORT || + err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) { + TEST_SAY("Ignoring expected error: %s: %s\n", + rd_kafka_err2name(err), reason); + return 0; + } + return 1; +} + +/*#################################################################################*/ + +/** + * @brief Close with acknowledge test scenarios + * + * Tests consumer close behavior with different commit modes and topologies. + * Verifies that acknowledged messages are not redelivered to a second + consumer. + */ +static void do_test_close_with_acknowledge(void) { + /** + * @brief Test configuration for close with acknowledge scenarios + */ + typedef struct { + const char *test_name; + int topic_cnt; + int partitions[MAX_TOPICS]; + int msgs_per_partition; + commit_mode_t commit_mode; + int ack_count; + } close_ack_test_config_t; + + /* Test matrix: 3 commit modes × 4 topologies = 12 tests */ + close_ack_test_config_t tests[] = { + /* 1 topic, 1 partition */ + {"close-1t1p-no-commit", 1, {1}, 20, COMMIT_MODE_NONE, 10}, + {"close-1t1p-commit-async", 1, {1}, 20, COMMIT_MODE_ASYNC, 10}, + {"close-1t1p-commit-sync", 1, {1}, 20, COMMIT_MODE_SYNC, 10}, + + /* 1 topic, multiple partitions */ + {"close-1t3p-no-commit", 1, {3}, 10, COMMIT_MODE_NONE, 20}, + {"close-1t3p-commit-async", 1, {3}, 10, COMMIT_MODE_ASYNC, 20}, + {"close-1t3p-commit-sync", 1, {3}, 10, COMMIT_MODE_SYNC, 15}, + + /* Multiple topics, 1 partition each */ + {"close-3t1p-no-commit", 3, {1, 1, 1}, 10, COMMIT_MODE_NONE, 20}, + {"close-3t1p-commit-async", + 3, + {1, 1, 1}, + 10, + COMMIT_MODE_ASYNC, + 20}, + {"close-3t1p-commit-sync", 3, {1, 1, 1}, 10, COMMIT_MODE_SYNC, 20}, + + /* Multiple topics, multiple partitions */ + {"close-2t2p-no-commit", 2, {2, 2}, 10, COMMIT_MODE_NONE, 20}, + {"close-2t2p-commit-async", 2, {2, 2}, 10, COMMIT_MODE_ASYNC, 20}, + {"close-2t2p-commit-sync", 2, {2, 2}, 10, COMMIT_MODE_SYNC, 20}, + }; + + SUB_TEST(); + + for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + close_ack_test_config_t *config = &tests[i]; + test_context_t ctx = {0}; + rd_kafka_share_t *c1, *c2; + tracked_msg_t tracked_msgs[BATCH_SIZE]; + int tracked_cnt = 0; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + TEST_SAY("\n========================================\n"); + TEST_SAY("Test: %s\n", config->test_name); + TEST_SAY("Topology: %d topic(s), partitions: [", + config->topic_cnt); + for (int j = 0; j < config->topic_cnt; j++) { + TEST_SAY("%d%s", config->partitions[j], + j < config->topic_cnt - 1 ? ", " : ""); + } + TEST_SAY("]\n"); + TEST_SAY("Commit mode: %s\n", + commit_mode_str(config->commit_mode)); + TEST_SAY("========================================\n\n"); + + ctx.group_id = "0178-group"; + setup_topics_and_produce(&ctx, config->topic_cnt, + config->partitions, + config->msgs_per_partition); + + /* Set group offset to earliest */ + test_share_set_auto_offset_reset(ctx.group_id, "earliest"); + + /* Create C1 with explicit ack mode (will call acknowledge() + * explicitly). C2 will be created after C1 closes. */ + c1 = create_share_consumer_with_receipts(ctx.group_id, + "explicit", &receipts); + + /* Subscribe C1 only - C2 will subscribe after C1 closes */ + subscribe_consumer(c1, ctx.topic_names, ctx.topic_cnt); + + /* C1: Consume and acknowledge */ + consume_and_track(c1, "C1", config->ack_count, rd_true, + tracked_msgs, &tracked_cnt); + + /* Execute commit */ + perform_commit(c1, "C1", config->commit_mode, &receipts); + + /* Close C1 - drains any outstanding ack callbacks + * synchronously, so receipts are settled by the time close + * returns. */ + TEST_SAY("C1: Closing consumer\n"); + rd_kafka_error_t *c1_close_err = + rd_kafka_share_consumer_close(c1); + TEST_ASSERT(c1_close_err == NULL, + "C1: close returned error: %s", + rd_kafka_error_string(c1_close_err)); + TEST_SAY("C1: Closed successfully\n"); + + /* Verify the union of callback + sync receipts equals the + * set of messages C1 acknowledged. */ + verify_receipts_match(&receipts, tracked_msgs, tracked_cnt, + commit_mode_str(config->commit_mode)); + + /* Create C2 with implicit ack mode after C1 is fully destroyed + */ + TEST_SAY("C2: Creating and subscribing after C1 close\n"); + c2 = test_create_share_consumer(ctx.group_id, "implicit"); + subscribe_consumer(c2, ctx.topic_names, ctx.topic_cnt); + + /* C2: Consume and verify no redelivery */ + verify_tracked_messages(c2, "C2", + ctx.total_msgs_produced - tracked_cnt, + tracked_msgs, tracked_cnt, rd_false, + commit_mode_str(config->commit_mode)); + + /* Close C2 */ + TEST_SAY("C2: Closing consumer\n"); + rd_kafka_share_consumer_close(c2); + TEST_SAY("C2: Closed successfully\n"); + test_share_destroy(c2); + + + /* Cleanup */ + free_tracked_messages(tracked_msgs, tracked_cnt); + ack_receipts_destroy(&receipts); + + for (int t = 0; t < ctx.topic_cnt; t++) { + rd_free(ctx.topic_names[t]); + } + + TEST_SAY("Test %s completed successfully\n\n", + config->test_name); + test_share_destroy(c1); + } + + SUB_TEST_PASS(); +} + +/** + * @brief Test close without acknowledge + * + * Verifies that when a consumer closes without acknowledging consumed records, + * those records are released back to the share group and can be consumed by + * another consumer. + * + * Test flow: + * 1. C1 consumes records but does NOT acknowledge them + * 2. C1 closes (should release the unacknowledged records) + * 3. C2 should be able to consume those same records + * + * Tests multiple topologies: 1t1p, 1t3p, 3t1p, 2t2p + */ +static void do_test_close_without_acknowledge() { + /** + * @brief Test configuration for close without acknowledge scenarios + */ + typedef struct { + const char *test_name; + int topic_cnt; + int partitions[MAX_TOPICS]; + int msgs_per_partition; + int track_count; + } close_no_ack_test_config_t; + + /* Test matrix: various topologies */ + close_no_ack_test_config_t tests[] = { + /* 1 topic, 1 partition */ + {"close-no-ack-1t1p", 1, {1}, 20, 5}, + + /* 1 topic, multiple partitions */ + {"close-no-ack-1t3p", 1, {3}, 10, 5}, + + /* Multiple topics, 1 partition each */ + {"close-no-ack-3t1p", 3, {1, 1, 1}, 10, 5}, + + /* Multiple topics, multiple partitions */ + {"close-no-ack-2t2p", 2, {2, 2}, 10, 5}, + }; + + SUB_TEST(); + + for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { + close_no_ack_test_config_t *config = &tests[i]; + test_context_t ctx = {0}; + rd_kafka_share_t *c1, *c2; + tracked_msg_t tracked_msgs[BATCH_SIZE]; + int tracked_cnt = 0; + char group_id[64]; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + TEST_SAY("\n========================================\n"); + TEST_SAY("Topology: %d topic(s), partitions: [", + config->topic_cnt); + for (int j = 0; j < config->topic_cnt; j++) { + TEST_SAY("%d%s", config->partitions[j], + j < config->topic_cnt - 1 ? ", " : ""); + } + TEST_SAY("]\n"); + TEST_SAY("========================================\n\n"); + + rd_snprintf(group_id, sizeof(group_id), + "0178-group-no-ack-%" PRIu64, test_id_generate()); + ctx.group_id = group_id; + setup_topics_and_produce(&ctx, config->topic_cnt, + config->partitions, + config->msgs_per_partition); + + test_share_set_auto_offset_reset(ctx.group_id, "earliest"); + + c1 = create_share_consumer_with_receipts(ctx.group_id, + "implicit", &receipts); + subscribe_consumer(c1, ctx.topic_names, ctx.topic_cnt); + + /* C1: Consume and track without acknowledging. */ + consume_and_track(c1, "C1", config->track_count, rd_false, + tracked_msgs, &tracked_cnt); + + /* Close C1 - unacked records must be released back to the + * share group. No acks were issued, so we expect zero + * receipts after close completes. */ + TEST_SAY("C1: Closing without acknowledging\n"); + rd_kafka_error_t *c1_close_err = + rd_kafka_share_consumer_close(c1); + TEST_ASSERT(c1_close_err == NULL, + "C1: close returned error: %s", + rd_kafka_error_string(c1_close_err)); + + /* No acks were issued, so the receipts sink must be empty. */ + verify_receipts_match(&receipts, NULL, 0, "no-ack-close"); + + c2 = test_create_share_consumer(ctx.group_id, "implicit"); + subscribe_consumer(c2, ctx.topic_names, ctx.topic_cnt); + + /* C2: Verify every tracked message is redelivered. Cap the + * scan at total produced so we don't loop forever if redelivery + * is broken. */ + verify_tracked_messages(c2, "C2", ctx.total_msgs_produced, + tracked_msgs, tracked_cnt, rd_true, + "no-ack-close"); + + TEST_SAY("C2: Closing consumer\n"); + rd_kafka_share_consumer_close(c2); + test_share_destroy(c2); + + free_tracked_messages(tracked_msgs, tracked_cnt); + ack_receipts_destroy(&receipts); + + for (int t = 0; t < ctx.topic_cnt; t++) { + rd_free(ctx.topic_names[t]); + } + + TEST_SAY("Test %s completed successfully\n\n", + config->test_name); + test_share_destroy(c1); + } + + SUB_TEST_PASS(); +} + +/** + * @brief Test close with slow broker response + * + * Verifies that close() waits for the broker to respond to the + * session close request (sent to partition leader), even when + * the broker is slow to respond. + * + * Tests 3 scenarios with a 3-broker setup: + * 1. Only broker 1 has delayed response (10s) + * 2. Brokers 1 and 2 have delayed responses (10s each) + * 3. All 3 brokers have delayed responses (10s each) + * + * In each case, close() should wait for the response and complete + * successfully after the delay. + */ +static void do_test_close_with_slow_broker_response(void) { + typedef struct { + const char *test_name; + int delayed_broker_cnt; + int32_t delayed_brokers[3]; + } delay_test_config_t; + + delay_test_config_t tests[] = { + {"1-broker-delayed", 1, {1, 0, 0}}, + {"2-brokers-delayed", 2, {1, 2, 0}}, + {"3-brokers-delayed", 3, {1, 2, 3}}, + }; + + for (size_t test_idx = 0; test_idx < sizeof(tests) / sizeof(tests[0]); + test_idx++) { + delay_test_config_t *config = &tests[test_idx]; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_share_t *consumer; + const int rtt_delay_ms = 10000; /* 10 seconds */ + rd_ts_t t_start, t_elapsed_ms; + int i; + + TEST_SAY("\n========================================\n"); + TEST_SAY("Test: %s\n", config->test_name); + TEST_SAY("Delayed brokers: %d\n", config->delayed_broker_cnt); + TEST_SAY("========================================\n\n"); + + SUB_TEST("%s", config->test_name); + + setup_3broker_share_consumer(config->test_name, 0, &mcluster, + &consumer); + + /* Inject RTT delay for ShareAck request on specified brokers; + * the session-close request */ + for (i = 0; i < config->delayed_broker_cnt; i++) { + TEST_ASSERT( + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, config->delayed_brokers[i], + RD_KAFKAP_ShareAcknowledge, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, + rtt_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to inject ShareAcknowledge delay"); + } + + TEST_SAY( + "Calling close() with delayed broker response(s)...\n"); + t_start = test_clock(); + rd_kafka_share_consumer_close(consumer); + t_elapsed_ms = (test_clock() - t_start) / 1000; + + TEST_SAY("Close completed after %" PRId64 " ms\n", + t_elapsed_ms); + + /* close() should wait ~rtt_delay_ms for the slowest delayed + * broker. Allow (rtt - 1s) to (rtt + 2s). */ + int64_t min_expected_ms = rtt_delay_ms - 1000; + int64_t max_expected_ms = rtt_delay_ms + 2000; + + TEST_ASSERT(t_elapsed_ms >= min_expected_ms && + t_elapsed_ms <= max_expected_ms, + "Close took %" PRId64 + " ms, expected between %" PRId64 " and %" PRId64 + " ms (with %d broker(s) delayed by %dms)", + t_elapsed_ms, min_expected_ms, max_expected_ms, + config->delayed_broker_cnt, rtt_delay_ms); + + TEST_SAY( + "SUCCESS: Close waited approximately %dms for " + "delayed broker response(s)\n", + rtt_delay_ms); + + test_share_destroy(consumer); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); + } +} + +/** + * @brief Test close respects socket timeout with very slow broker(s). + * + * Verifies that when broker(s) take longer than socket.timeout.ms to + * respond to the session close request, close() times out at + * socket.timeout.ms and does not wait for the full broker delay. + * + * Tests 3 scenarios with a 3-broker setup: + * 1. Only broker 1 has delayed response (40s, exceeds 20s socket timeout) + * 2. Brokers 1 and 2 have delayed responses + * 3. All 3 brokers have delayed responses + */ +static void do_test_close_respects_socket_timeout(void) { + typedef struct { + const char *test_name; + int delayed_broker_cnt; + int32_t delayed_brokers[3]; + } delay_test_config_t; + + delay_test_config_t tests[] = { + {"close-timeout-1-broker", 1, {1, 0, 0}}, + {"close-timeout-2-brokers", 2, {1, 2, 0}}, + {"close-timeout-3-brokers", 3, {1, 2, 3}}, + }; + + const int rtt_delay_ms = 40000; /* 40s — exceeds socket.timeout */ + const int socket_timeout_ms = 20000; /* 20s */ + + for (size_t test_idx = 0; test_idx < sizeof(tests) / sizeof(tests[0]); + test_idx++) { + delay_test_config_t *config = &tests[test_idx]; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_share_t *consumer; + rd_ts_t t_start, t_elapsed_ms; + int i; + + TEST_SAY("\n========================================\n"); + TEST_SAY("Test: %s\n", config->test_name); + TEST_SAY( + "Delayed brokers: %d, RTT: %dms, Socket timeout: %dms\n", + config->delayed_broker_cnt, rtt_delay_ms, + socket_timeout_ms); + TEST_SAY("========================================\n\n"); + + SUB_TEST("%s", config->test_name); + + setup_3broker_share_consumer( + config->test_name, socket_timeout_ms, &mcluster, &consumer); + + for (i = 0; i < config->delayed_broker_cnt; i++) { + TEST_SAY( + "Injecting %dms RTT delay on broker %d " + "(exceeds socket timeout %dms)\n", + rtt_delay_ms, config->delayed_brokers[i], + socket_timeout_ms); + rd_kafka_mock_broker_set_rtt( + mcluster, config->delayed_brokers[i], rtt_delay_ms); + } + + TEST_SAY("Calling close() - should timeout around %dms...\n", + socket_timeout_ms); + t_start = test_clock(); + rd_kafka_share_consumer_close(consumer); + t_elapsed_ms = (test_clock() - t_start) / 1000; + + TEST_SAY("Close completed after %" PRId64 " ms\n", + t_elapsed_ms); + + /* close() should timeout around socket.timeout.ms. + * Allow ±5s margin. */ + int64_t min_expected_ms = socket_timeout_ms - 5000; + int64_t max_expected_ms = socket_timeout_ms + 5000; + + TEST_ASSERT(t_elapsed_ms >= min_expected_ms && + t_elapsed_ms <= max_expected_ms, + "Close took %" PRId64 + " ms, expected ~%dms (between %" PRId64 + " and %" PRId64 + " ms). Should NOT wait for full broker delay " + "of %dms", + t_elapsed_ms, socket_timeout_ms, min_expected_ms, + max_expected_ms, rtt_delay_ms); + + /* Defensively verify it did not wait the full broker delay. */ + TEST_ASSERT(t_elapsed_ms < (rtt_delay_ms - 10000), + "Close took %" PRId64 + " ms — too close to broker delay %dms. " + "Should have timed out at socket.timeout.ms=%dms", + t_elapsed_ms, rtt_delay_ms, socket_timeout_ms); + + TEST_SAY( + "SUCCESS: Close respected socket timeout (%dms) with " + "%d delayed broker(s)\n", + socket_timeout_ms, config->delayed_broker_cnt); + + test_share_destroy(consumer); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); + } +} + +/** + * @brief Test close with broker error response. + * + * Verifies that close() handles error responses from the broker + * gracefully. When the broker(s) respond with an error to the + * ShareAcknowledge request (session close request), close() should + * complete without retrying. + * + * Tests 3 scenarios with a 3-broker setup: + * 1. Only broker 1 returns LEADER_NOT_AVAILABLE + * 2. Brokers 1 and 2 return the error + * 3. All 3 brokers return the error + */ +static void do_test_close_with_broker_error_response(void) { + typedef struct { + const char *test_name; + int erroring_broker_cnt; + int32_t erroring_brokers[3]; + } error_test_config_t; + + error_test_config_t tests[] = { + {"close-error-1-broker", 1, {1, 0, 0}}, + {"close-error-2-brokers", 2, {1, 2, 0}}, + {"close-error-3-brokers", 3, {1, 2, 3}}, + }; + + const rd_kafka_resp_err_t error_code = + RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE; + + for (size_t test_idx = 0; test_idx < sizeof(tests) / sizeof(tests[0]); + test_idx++) { + error_test_config_t *config = &tests[test_idx]; + rd_kafka_mock_cluster_t *mcluster; + rd_kafka_share_t *consumer; + int i; + + TEST_SAY("\n========================================\n"); + TEST_SAY("Test: %s\n", config->test_name); + TEST_SAY("Erroring brokers: %d, error code: %s\n", + config->erroring_broker_cnt, + rd_kafka_err2str(error_code)); + TEST_SAY("========================================\n\n"); + + SUB_TEST("%s", config->test_name); + + setup_3broker_share_consumer(config->test_name, 0, &mcluster, + &consumer); + + /* Push the error onto the specified brokers' next + * ShareAcknowledge request. */ + for (i = 0; i < config->erroring_broker_cnt; i++) { + TEST_SAY("Injecting %s error on broker %d\n", + rd_kafka_err2str(error_code), + config->erroring_brokers[i]); + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, config->erroring_brokers[i], + RD_KAFKAP_ShareAcknowledge, 1, error_code, 0); + } + + TEST_SAY( + "Calling close() - should complete despite error(s)...\n"); + rd_kafka_share_consumer_close(consumer); + TEST_SAY("Close completed\n"); + + TEST_SAY( + "SUCCESS: Close handled error %s gracefully on %d " + "broker(s) (no retry)\n", + rd_kafka_err2str(error_code), config->erroring_broker_cnt); + + test_share_destroy(consumer); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); + } +} + + +/** + * @brief Test close() when brokers have pending acks queued. + * + * Setup: + * - 3-broker mock cluster, topic with 3 partitions (one leader per broker). + * - Inject 5s RTT delay on ShareAcknowledge for all brokers — this makes + * every ack the consumer sends take 5s broker-side. + * - Produce 30 msgs (10/partition), consume them all with an implicit-ack + * consumer. Implicit-ack piggybacks acks on the next ShareFetch, so by + * the end of the first round there are still acks in flight or queued. + * - Repeat: produce another 30, consume them all (same consumer). + * - Call close(). close() should drain the in-flight ShareAcknowledges + * and send the session-leave; with the 5s delay this should take + * roughly 5s (remaining ack RTT) + 5s (leave RTT) = ~10s. + * - Remove the RTT delay, start a second consumer (also implicit ack) + * and verify it receives 0 messages over up to 5 fetch attempts. If + * any message is delivered, close() failed to send the acks. + */ +static void do_test_close_with_broker_busy(void) { + const char *test_name = "close-broker-busy"; + const int partition_cnt = 3; + const int msgs_per_partition = 10; + const int total_msgs = partition_cnt * msgs_per_partition; + const int rtt_delay_ms = 5000; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *consumer, *c2; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error, *close_err; + char topic[64], group[64], errstr[512]; + int iter, p, i; + size_t rcvd; + rd_ts_t t_start, t_elapsed_ms; + + SUB_TEST("%s", test_name); + + rd_snprintf(topic, sizeof(topic), "mock-%s", test_name); + rd_snprintf(group, sizeof(group), "sg-%s", test_name); + + TEST_SAY("\n========================================\n"); + TEST_SAY("Test: %s\n", test_name); + TEST_SAY("ShareAck RTT: %dms on all brokers\n", rtt_delay_ms); + TEST_SAY("========================================\n\n"); + + /* Cluster + APIs + topic */ + mcluster = test_mock_cluster_new(3, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, partition_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Implicit-ack consumer */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "implicit"); + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create consumer: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, subs), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(subs); + + /* Inject 5s RTT delay on the next 20 ShareAcknowledge responses on + * each broker (no error, just delay) */ + TEST_SAY( + "Injecting %dms RTT delay on ShareAcknowledge " + "(20 entries/broker)\n", + rtt_delay_ms); + for (i = 1; i <= 3; i++) { + int j; + for (j = 0; j < 20; j++) { + TEST_ASSERT( + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, i, RD_KAFKAP_ShareAcknowledge, 1, + (rd_kafka_resp_err_t)0, + rtt_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to push RTT delay on broker %d", i); + } + } + + /* Two rounds of produce+consume so that by the end there are acks + * queued/in-flight on every broker. */ + for (iter = 1; iter <= 2; iter++) { + int consumed = 0, attempts = 0; + TEST_SAY( + "Iteration %d: producing %d msgs across %d " + "partitions\n", + iter, total_msgs, partition_cnt); + for (p = 0; p < partition_cnt; p++) { + test_produce_msgs_easy_v( + topic, 0, p, iter * msgs_per_partition, + msgs_per_partition, 16, "bootstrap.servers", + bootstraps, NULL); + } + + TEST_SAY("Iteration %d: consuming %d msgs\n", iter, total_msgs); + while (consumed < total_msgs && attempts++ < 30) { + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < (int)rcvd; i++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, i); + if (rkm && !rkm->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(rd_kafka_share_commit_async(consumer) == + NULL, + "fail to call commit async"); + } + TEST_ASSERT(consumed == total_msgs, + "Iteration %d: expected %d msgs, got %d", iter, + total_msgs, consumed); + TEST_SAY("Iteration %d: consumed all %d msgs\n", iter, + consumed); + } + + /* Call close. Expect ~10s: ~5s for in-flight ack to drain, then + * ~5s for the session-leave ShareAcknowledge to come back. */ + TEST_SAY("Calling close() - expecting ~10s\n"); + t_start = test_clock(); + close_err = rd_kafka_share_consumer_close(consumer); + t_elapsed_ms = (test_clock() - t_start) / 1000; + + TEST_SAY("Close completed after %" PRId64 " ms\n", t_elapsed_ms); + TEST_ASSERT(close_err == NULL, "Close returned error: %s", + rd_kafka_error_string(close_err)); + TEST_ASSERT(t_elapsed_ms >= 8000 && t_elapsed_ms <= 12000, + "Close took %" PRId64 " ms, expected 8000-12000 ms", + t_elapsed_ms); + + test_share_destroy(consumer); + + /* Verify nothing was redelivered: c2 should consume 0 messages + * because c1's close() flushed all acks. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "implicit"); + c2 = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(c2, "Failed to create c2: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(c2, subs), "c2 subscribe failed"); + rd_kafka_topic_partition_list_destroy(subs); + + TEST_SAY("c2: verifying no msgs redelivered (5 fetch attempts)\n"); + for (i = 0; i < 5; i++) { + error = rd_kafka_share_poll(c2, 2000, &batch); + if (error) { + TEST_SAY("c2 attempt %d: error: %s\n", i + 1, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT( + rcvd == 0, + "c2 attempt %d: received %d msgs (expected 0 — close " + "did not flush acks)", + i + 1, (int)rcvd); + } + + TEST_SAY("SUCCESS: close() flushed acks; c2 received no msgs\n"); + + test_share_destroy(c2); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test close() when the broker is down. + * + * Verifies that close() returns immediately without an error + * when the broker is down + * + * Setup: + * - Consumer subscribes, consumes at least one batch, then we set the + * broker down and call close(). + * - close() should return immediately and return NULL (no error). + */ +static void do_test_close_with_broker_down(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *consumer; + rd_kafka_conf_t *conf; + const char *topic = "mock-close-broker-down"; + const char *group = "sg-close-broker-down"; + const int msgcnt = 10; + const int socket_timeout_ms = 20000; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + rd_kafka_error_t *close_err; + size_t rcvd; + int attempts = 0; + rd_bool_t got_msgs = rd_false; + rd_ts_t t_start, t_elapsed_ms; + char errstr[512]; + + SUB_TEST("close-with-broker-down"); + + /* Suppress the transport-error / all-brokers-down events that the + * test framework's default error_cb would otherwise fail on once we + * set the broker down. Scoped to this test only. */ + test_curr->is_fatal_cb = test_close_with_broker_down_is_fatal_cb; + + TEST_SAY("\n========================================\n"); + TEST_SAY("Test: close with broker down\n"); + TEST_SAY("Socket timeout: %dms\n", socket_timeout_ms); + TEST_SAY("========================================\n\n"); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Produce msgcnt messages via test helper. */ + test_produce_msgs_easy_v(topic, 0, 0, 0, msgcnt, 16, + "bootstrap.servers", bootstraps, NULL); + + /* Consumer with implicit ack mode and explicit socket timeout. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "implicit"); + + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create consumer: %s", errstr); + + rd_kafka_topic_partition_list_t *subs = + rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, subs), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume until we get at least one batch of messages, then break. + */ + TEST_SAY("Consuming until first non-empty batch...\n"); + while (!got_msgs && attempts++ < 30) { + error = rd_kafka_share_poll(consumer, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + if (rcvd > 0) { + TEST_SAY("Received %d messages\n", (int)rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + got_msgs = rd_true; + break; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(got_msgs, "Expected to receive at least one message"); + + /* Bring the broker down before calling close(). */ + TEST_SAY("Setting broker 1 down\n"); + TEST_ASSERT(rd_kafka_mock_broker_set_down(mcluster, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set broker down"); + + /* close() should fail fast (broker connection was closed) */ + TEST_SAY("Calling close() with broker down\n"); + t_start = test_clock(); + close_err = rd_kafka_share_consumer_close(consumer); + t_elapsed_ms = (test_clock() - t_start) / 1000; + + TEST_SAY("Close completed after %" PRId64 " ms (err=%s)\n", + t_elapsed_ms, + close_err ? rd_kafka_error_string(close_err) : "NULL"); + + if (close_err) + rd_kafka_error_destroy(close_err); + + TEST_SAY("SUCCESS: close() returned in %" PRId64 + "ms after broker down\n", + t_elapsed_ms); + + /* Bring the broker back up so the consumer's network threads can + * finish their shutdown handshake cleanly during destroy. */ + TEST_ASSERT(rd_kafka_mock_broker_set_up(mcluster, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set broker up"); + + test_share_destroy(consumer); + test_mock_cluster_destroy(mcluster); + + /* Restore default error-fatal behavior for subsequent tests. */ + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + +/** + * @brief Test: calling share-consumer APIs after close() completes. + * + * Verifies every guarded API returns RD_KAFKA_RESP_ERR__STATE with + * "closed" in the error string. + */ +static void do_test_api_calls_on_closed_consumer(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_conf_t *conf; + rd_kafka_share_t *consumer; + const char *topic = "0178-api-after-close"; + const char *group = "grp-0178-api-after-close"; + char errstr[512]; + rd_kafka_error_t *close_error; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Create mock topic"); + + /* Produce a single message via test helper (creates an internal + * producer for the duration of the call). */ + test_produce_msgs_easy_v(topic, 0, 0, 0, 1, 3, "bootstrap.servers", + bootstraps, NULL); + + /* Consumer */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "consumer: %s", errstr); + + rd_kafka_topic_partition_list_t *subs = + rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, subs), "subscribe"); + rd_kafka_topic_partition_list_destroy(subs); + + /* Close (blocking), then verify all APIs reject further calls. */ + TEST_SAY("Calling close() and waiting for completion\n"); + close_error = rd_kafka_share_consumer_close(consumer); + TEST_ASSERT(close_error == NULL, "initial close: %s", + close_error ? rd_kafka_error_string(close_error) : ""); + TEST_ASSERT(rd_kafka_share_consumer_closed(consumer), + "consumer should report closed"); + + TEST_SAY("Exercising APIs on closed consumer\n"); + verify_all_apis_return_error(consumer, topic); + + test_share_destroy(consumer); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test: once close_queue() returns successfully, all subsequent API + * calls must be rejected with RD_KAFKA_RESP_ERR__STATE. + * + * A 5s RTT delay is injected on ShareAcknowledge so the consumer remains + * in the "closing" state (close_queue returns immediately, actual close + * completes later), giving us a deterministic window to exercise every + * guarded API. + */ +static void do_test_api_calls_during_closing(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_conf_t *conf; + rd_kafka_share_t *consumer; + const char *topic = "0178-api-during-closing"; + const char *group = "grp-0178-api-during-closing"; + char errstr[512]; + rd_kafka_queue_t *queue; + rd_kafka_error_t *close_error; + const int rtt_delay_ms = 5000; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Create mock topic"); + + /* Produce one message via test helper. */ + test_produce_msgs_easy_v(topic, 0, 0, 0, 1, 3, "bootstrap.servers", + bootstraps, NULL); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "consumer: %s", errstr); + + rd_kafka_topic_partition_list_t *subs = + rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, subs), "subscribe"); + rd_kafka_topic_partition_list_destroy(subs); + + /* Consume one batch so there's state to flush on close. */ + { + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + int attempts = 0; + rd_kafka_error_t *cerr = NULL; + while (attempts++ < 20 && rcvd == 0) { + cerr = rd_kafka_share_poll(consumer, 500, &batch); + if (cerr) { + rd_kafka_error_destroy(cerr); + continue; + } + rcvd = rd_kafka_messages_count(batch); + if (rcvd == 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + + /* Inject 5s delay on ShareAcknowledge so close() stays in flight. */ + TEST_SAY("Injecting %dms RTT delay on broker\n", rtt_delay_ms); + rd_kafka_mock_broker_set_rtt(mcluster, 1, rtt_delay_ms); + + queue = rd_kafka_queue_new(test_share_consumer_get_rk(consumer)); + + TEST_SAY("Calling close_queue() to initiate async close\n"); + close_error = rd_kafka_share_consumer_close_queue(consumer, queue); + TEST_ASSERT(close_error == NULL, "close_queue: %s", + close_error ? rd_kafka_error_string(close_error) : ""); + + /* close_queue has returned. Consumer must now be in closing state + * (not fully closed yet, since ShareAcknowledge response is still + * delayed). Every subsequent API call must be rejected. */ + TEST_ASSERT(!rd_kafka_share_consumer_closed(consumer), + "consumer should still be closing, not closed"); + + TEST_SAY("Exercising APIs on closing consumer\n"); + verify_all_apis_return_error(consumer, topic); + + /* Wait for async close to complete before destroying. */ + TEST_SAY("Waiting for consumer to finish closing\n"); + while (!rd_kafka_share_consumer_closed(consumer)) { + rd_usleep(500 * 1000, NULL); /* 500ms */ + } + + rd_kafka_queue_destroy(queue); + test_share_destroy(consumer); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief Test that the share-ack callback fires for un-committed implicit + * acks during close. + * + * In implicit mode, records consumed in the first poll are auto-acked via + * piggyback on the next ShareFetch. After polling twice (first to fetch, + * second to drive the piggyback ack) and closing without an explicit + * commit, the ack callback must have fired with the consumed offsets. + */ +static void do_test_implicit_ack_callback_fires_on_close(void) { + const char *topic_name; + const char *group; + char group_id[64]; + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *raw_batch = NULL; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_error_t *error; + ack_receipts_t receipts; + size_t rcvd, j; + int consumed = 0; + int attempts = 0; + + SUB_TEST(); + + ack_receipts_init(&receipts); + + topic_name = test_mk_topic_name("0178-impl-ack-on-close", 1); + test_create_topic_wait_exists(common_admin, topic_name, 1, -1, + 60 * 1000); + test_produce_msgs_simple(common_producer, topic_name, 0, 1); + + rd_snprintf(group_id, sizeof(group_id), + "0178-group-impl-ack-close-%" PRIu64, test_id_generate()); + group = group_id; + + test_share_set_auto_offset_reset(group, "earliest"); + rkshare = + create_share_consumer_with_receipts(group, "implicit", &receipts); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic_name, + RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + + /* First poll: fetch the record (auto-ack pending for next poll) */ + while (consumed < 1 && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 2000, &raw_batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(raw_batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(raw_batch, j); + if (rkm && !rkm->err) + consumed++; + } + rd_kafka_messages_destroy(raw_batch); + raw_batch = NULL; + } + TEST_ASSERT(consumed == 1, "Expected 1 record, got %d", consumed); + + /* Second poll: drives the piggybacked implicit ack to the broker */ + attempts = 5; + while (attempts-- > 0) { + error = rd_kafka_share_poll(rkshare, 1000, &raw_batch); + if (error) + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(raw_batch); + raw_batch = NULL; + } + + /* Close - any pending ack-completion callbacks must be flushed */ + TEST_SAY("Closing share consumer without explicit commit\n"); + error = rd_kafka_share_consumer_close(rkshare); + TEST_ASSERT(error == NULL, "close returned error: %s", + rd_kafka_error_string(error)); + + TEST_SAY( + "After close: callback_invocations=%d, receipts=%d, " + "last_cb_err=%s\n", + receipts.callback_invocations, receipts.receipt_cnt, + rd_kafka_err2name(receipts.last_cb_err)); + + TEST_ASSERT(receipts.callback_invocations >= 1, + "Expected ack callback to fire at least once " + "by the time close returns, got %d invocations", + receipts.callback_invocations); + TEST_ASSERT(receipts.receipt_cnt >= 1, + "Expected at least 1 acked offset reported via " + "the callback by close, got %d", + receipts.receipt_cnt); + TEST_ASSERT(receipts.last_cb_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected NO_ERROR in ack callback, got %s", + rd_kafka_err2name(receipts.last_cb_err)); + + ack_receipts_destroy(&receipts); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + + +int main_0178_share_consumer_close(int argc, char **argv) { + /* Set overall timeout for all tests */ + test_timeout_set(600); + + /* Create common handles reused across all real-broker tests */ + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + /* Real broker tests */ + do_test_close_with_acknowledge(); + do_test_close_without_acknowledge(); + do_test_implicit_ack_callback_fires_on_close(); + + /* Cleanup common handles */ + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} + +int main_0178_share_consumer_close_local(int argc, char **argv) { + /* Mock broker tests only (no real broker needed) */ + TEST_SKIP_MOCK_CLUSTER(0); + test_timeout_set(300); + + do_test_close_with_slow_broker_response(); + do_test_close_respects_socket_timeout(); + do_test_close_with_broker_error_response(); + do_test_close_with_broker_busy(); + do_test_close_with_broker_down(); + do_test_api_calls_on_closed_consumer(); + do_test_api_calls_during_closing(); + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0179-share_consumer_destroy.c b/lib/librdkafka-2.15.0/tests/0179-share_consumer_destroy.c new file mode 100644 index 00000000000..ff684b483be --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0179-share_consumer_destroy.c @@ -0,0 +1,2482 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "../src/rdkafka.h" +#include "../src/rdkafka_proto.h" + +#define CONSUME_ARRAY 1000 +#define TEST_MSGS 100 +#define MAX_CONSUME_ATTEMPTS 30 +#define MAX_PARTITIONS 32 + +/* Shared producer/admin handles for real-broker tests. Created in + * main_0179_share_consumer_destroy and reused across sub-tests. */ +static rd_kafka_t *common_producer; +static rd_kafka_t *common_admin; + + +/**************************************************************************** + * Acknowledgement Callback Tracking Helpers + ****************************************************************************/ + +/** + * @brief Create a share consumer for the mock-broker tests in this file. + * + * Points at the supplied mock bootstraps (test_create_share_consumer + * cannot, since it reads the real-broker bootstrap globals), sets + * a fast topic.metadata.refresh.interval.ms (500) so the client + * observes mock metadata changes quickly, and conditionally enables + * explicit ack mode. + */ +/** + * @brief One per-partition entry from a share-ack callback invocation. + * Each callback may report N partition entries; we record one + * ack_receipt_t per (topic, partition) entry, carrying the + * callback-level err and the offsets count for that partition. + */ +typedef struct { + char *topic; + int32_t partition; + int offset_cnt; + rd_kafka_resp_err_t err; +} ack_receipt_t; + +typedef struct { + ack_receipt_t *receipts; + int receipt_cnt; + int receipt_capacity; + int callback_invocations; +} ack_receipts_t; + +static void ack_receipts_init(ack_receipts_t *r) { + memset(r, 0, sizeof(*r)); +} + +static void ack_receipts_destroy(ack_receipts_t *r) { + int i; + for (i = 0; i < r->receipt_cnt; i++) + rd_free(r->receipts[i].topic); + if (r->receipts) + rd_free(r->receipts); +} + +static void ack_receipts_add(ack_receipts_t *r, + const char *topic, + int32_t partition, + int offset_cnt, + rd_kafka_resp_err_t err) { + if (r->receipt_cnt == r->receipt_capacity) { + int new_cap = + r->receipt_capacity ? r->receipt_capacity * 2 : 16; + r->receipts = + rd_realloc(r->receipts, new_cap * sizeof(*r->receipts)); + r->receipt_capacity = new_cap; + } + r->receipts[r->receipt_cnt].topic = rd_strdup(topic); + r->receipts[r->receipt_cnt].partition = partition; + r->receipts[r->receipt_cnt].offset_cnt = offset_cnt; + r->receipts[r->receipt_cnt].err = err; + r->receipt_cnt++; +} + +/** + * @brief Share-ack callback that funnels each partition entry into + * the provided ack_receipts_t (passed as opaque). + */ +static void record_share_ack_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *parts, + rd_kafka_resp_err_t err, + void *opaque) { + ack_receipts_t *r = opaque; + size_t pcnt, p; + + (void)rkshare; + + r->callback_invocations++; + + pcnt = rd_kafka_share_partition_offsets_list_count(parts); + TEST_SAY("ack_cb invocation #%d: err=%s partitions=%zu\n", + r->callback_invocations, rd_kafka_err2name(err), pcnt); + + for (p = 0; p < pcnt; p++) { + const rd_kafka_share_partition_offsets_t *entry = + rd_kafka_share_partition_offsets_list_get(parts, p); + const rd_kafka_topic_partition_t *tp; + int ocnt; + if (!entry) + continue; + tp = rd_kafka_share_partition_offsets_partition(entry); + ocnt = rd_kafka_share_partition_offsets_offsets_cnt(entry); + TEST_SAY(" %s [%" PRId32 "] offsets=%d err=%s\n", tp->topic, + tp->partition, ocnt, rd_kafka_err2name(err)); + ack_receipts_add(r, tp->topic, tp->partition, ocnt, err); + } +} + +/** + * @brief One expected (topic, partition, err) group. The assertion + * is: across all recorded receipts whose + * (topic, partition, err) match this entry, the sum of + * offset_cnt equals expected_offset_cnt. Multiple receipts + * for the same (topic, partition, err) — possible if the + * callback fires more than once for the same partition — are + * summed. + * + * Receipts that don't match any expected group cause a test + * failure. + */ +typedef struct { + const char *topic; + int32_t partition; + rd_kafka_resp_err_t err; + int expected_offset_cnt; +} expected_ack_t; + +static void verify_ack_receipts(ack_receipts_t *r, + const expected_ack_t *expected, + int expected_cnt, + const char *label) { + int i, j; + + TEST_SAY( + "Verifying ack receipts (%s): %d receipts from %d " + "invocation(s), %d expected (topic, partition, err) group(s)\n", + label, r->receipt_cnt, r->callback_invocations, expected_cnt); + + /* Assert each expected group is satisfied. */ + for (j = 0; j < expected_cnt; j++) { + int observed_offset_cnt = 0; + int matching_receipts = 0; + + for (i = 0; i < r->receipt_cnt; i++) { + if (r->receipts[i].partition == expected[j].partition && + r->receipts[i].err == expected[j].err && + strcmp(r->receipts[i].topic, expected[j].topic) == + 0) { + observed_offset_cnt += + r->receipts[i].offset_cnt; + matching_receipts++; + } + } + + if (observed_offset_cnt != expected[j].expected_offset_cnt) { + TEST_FAIL( + "ack receipts (%s): expected %d offsets for " + "%s [%" PRId32 + "] with err=%s, got %d (across " + "%d receipt(s))", + label, expected[j].expected_offset_cnt, + expected[j].topic, expected[j].partition, + rd_kafka_err2name(expected[j].err), + observed_offset_cnt, matching_receipts); + } + + TEST_SAY(" OK: %s [%" PRId32 + "] err=%s -> %d offsets across %d receipt(s)\n", + expected[j].topic, expected[j].partition, + rd_kafka_err2name(expected[j].err), + observed_offset_cnt, matching_receipts); + } + + /* Flag any receipt that didn't match any expected group. */ + for (i = 0; i < r->receipt_cnt; i++) { + rd_bool_t matched = rd_false; + for (j = 0; j < expected_cnt; j++) { + if (r->receipts[i].partition == expected[j].partition && + r->receipts[i].err == expected[j].err && + strcmp(r->receipts[i].topic, expected[j].topic) == + 0) { + matched = rd_true; + break; + } + } + if (!matched) { + TEST_FAIL( + "ack receipts (%s): unexpected receipt for " + "%s [%" PRId32 "] err=%s offsets=%d", + label, r->receipts[i].topic, + r->receipts[i].partition, + rd_kafka_err2name(r->receipts[i].err), + r->receipts[i].offset_cnt); + } + } +} + +/** + * @brief Drain pending ack callbacks from rk_rep by polling. + * + * Polls in 1s ticks for up to \p max_seconds. Stops early if + * the callback invocation counter advances past + * \p expected_invocations. + */ +static void drain_ack_callbacks(rd_kafka_share_t *rkshare) { + int max_seconds = 10; + rd_kafka_t *rk = test_share_consumer_get_rk(rkshare); + int waited; + for (waited = 0; waited < max_seconds; waited++) { + rd_kafka_poll(rk, 0); + rd_sleep(1); + } + /* One final poll for any callback that fired during the + * last sleep. */ + rd_kafka_poll(rk, 0); +} + +/**************************************************************************** + * General Helpers + ****************************************************************************/ + +static rd_kafka_share_t * +new_share_consumer_for_mock_test(const char *bootstraps, + const char *group_id, + rd_bool_t explicit_ack, + ack_receipts_t *receipts) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *consumer; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500"); + if (explicit_ack) + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + + consumer = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(consumer != NULL, "Failed to create share consumer"); + + if (receipts) { + rd_kafka_error_t *error = + rd_kafka_share_set_acknowledgement_commit_cb( + consumer, record_share_ack_cb, receipts); + TEST_ASSERT(error == NULL, + "Failed to set acknowledgement commit callback: " + "%s", + rd_kafka_error_string(error)); + } + return consumer; +} + + +/** + * @brief Create a share consumer for the real-broker tests in this file. + * + * Like test_create_share_consumer but additionally wires the + * ack-callback (record_share_ack_cb) when @p receipts is + * non-NULL. + */ +static rd_kafka_share_t * +new_share_consumer_for_real_test(const char *group_id, + const char *ack_mode, + ack_receipts_t *receipts) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *consumer; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "share.acknowledgement.mode", ack_mode, errstr, + sizeof(errstr)); + + consumer = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(consumer, "Failed to create share consumer: %s", errstr); + + if (receipts) { + rd_kafka_error_t *error = + rd_kafka_share_set_acknowledgement_commit_cb( + consumer, record_share_ack_cb, receipts); + TEST_ASSERT(error == NULL, + "Failed to set acknowledgement commit callback: " + "%s", + rd_kafka_error_string(error)); + } + return consumer; +} + +/** + * @brief Subscribe to topics. + */ +static void subscribe_topics(rd_kafka_share_t *consumer, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *tpl = + rd_kafka_topic_partition_list_new(topic_cnt); + for (int i = 0; i < topic_cnt; i++) { + rd_kafka_topic_partition_list_add(tpl, topics[i], + RD_KAFKA_PARTITION_UA); + } + TEST_ASSERT(!rd_kafka_share_subscribe(consumer, tpl), + "Subscribe failed"); + rd_kafka_topic_partition_list_destroy(tpl); +} + + +/** + * @brief Acknowledge messages in [start, end) and log each result. + */ +static void ack_range_logged(rd_kafka_share_t *rkshare, + rd_kafka_messages_t *batch, + int start, + int end, + int rcvd) { + int i; + for (i = start; i < end && i < rcvd; i++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(batch, i); + rd_kafka_resp_err_t ack_err; + if (!rkm) + continue; + ack_err = rd_kafka_share_acknowledge(rkshare, rkm); + TEST_SAY(" ack msg[%d] %s[%" PRId32 "]@%" PRId64 " -> %s\n", i, + rd_kafka_topic_name(rkm->rkt), rkm->partition, + rkm->offset, rd_kafka_err2str(ack_err)); + } +} + +/** + * @brief Destroy a share consumer, using flags variant when non-zero. + */ +static void destroy_share_consumer(rd_kafka_share_t *rkshare, + int destroy_flags) { + TEST_SAY("Calling destroy with flags 0x%x\n", destroy_flags); + if (destroy_flags) + rd_kafka_share_destroy_flags(rkshare, destroy_flags); + else + rd_kafka_share_destroy(rkshare); + TEST_SAY("Successfully destroyed share consumer\n"); +} + +/* Per-thread argument for destroy_watchdog_thread. Atomic `done` lets the + * main thread poll for completion without locking. */ +typedef struct destroy_watchdog_arg_s { + rd_kafka_share_t *rkshare; + int destroy_flags; + rd_atomic32_t done; +} destroy_watchdog_arg_t; + +static int destroy_watchdog_thread(void *p) { + destroy_watchdog_arg_t *a = p; + if (a->destroy_flags) + rd_kafka_share_destroy_flags(a->rkshare, a->destroy_flags); + else + rd_kafka_share_destroy(a->rkshare); + rd_atomic32_set(&a->done, 1); + return 0; +} + +/** + * @brief is_fatal_cb hook for test_broker_decommission_with_commit_sync. + * + * The decommission of a broker mid-flight produces __TRANSPORT and + * __ALL_BROKERS_DOWN errors as the connection is dropped and the client + * tries to reconnect. These are expected and should not fail the test. + */ +static int decommission_is_fatal_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + const char *reason) { + if (err == RD_KAFKA_RESP_ERR__TRANSPORT || + err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) { + TEST_SAY("Ignoring expected error: %s: %s\n", + rd_kafka_err2name(err), reason); + return 0; + } + return 1; +} + +/**************************************************************************** + * Test Cases + ****************************************************************************/ + +/** + * @brief This test uses mock brokers to simulate delayed broker responses and + * makes commit* calls causing acknowledgements to get cached. Eventually, calls + * destroy() to validate that it does not hang. + * @param destroy_flags 0 for normal destroy, + * RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE to skip consumer close. + * + * In case of RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, a __DESTROY error is + * expected to be returned from the broker thread having requests in-flight + */ +static void +test_destroy_with_cached_acks_and_delayed_broker(int destroy_flags) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + rd_kafka_topic_partition_list_t *commit_result = NULL; + const char *topic = "0179-destroy-cached-acks-delayed-broker"; + const char *group = "0179-destroy-cached-acks"; + const int broker_delay_ms = 5000; + size_t rcvd = 0; + int attempts = 0; + int i; + rd_ts_t t_start, t_elapsed_ms; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + TEST_SAY("Producing %d messages to topic %s\n", TEST_MSGS, topic); + test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS, 16, + "bootstrap.servers", bootstraps, NULL); + + TEST_SAY("Creating share consumer with explicit ack\n"); + rkshare = new_share_consumer_for_mock_test(bootstraps, group, rd_true, + &receipts); + subscribe_topics(rkshare, &topic, 1); + + TEST_SAY("Consuming messages (up to %d attempts)\n", + MAX_CONSUME_ATTEMPTS); + while (rcvd < 10 && attempts < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } else { + rcvd = rd_kafka_messages_count(rkmessages); + if (rcvd > 0) + TEST_SAY("Attempt %d: consumed %d messages\n", + attempts, (int)rcvd); + } + attempts++; + } + + TEST_ASSERT(rcvd >= 10, + "Expected at least 10 messages after %d attempts, got %d", + MAX_CONSUME_ATTEMPTS, (int)rcvd); + TEST_SAY("Successfully consumed %d messages\n", (int)rcvd); + + /* Inject %dms delay on the next 3 ShareAcknowledge responses on + * broker 1 (the only broker in this mock cluster). */ + TEST_SAY( + "Injecting %dms delay on the next 3 ShareAcknowledge " + "responses on broker 1\n", + broker_delay_ms); + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareAcknowledge, 3, + RD_KAFKA_RESP_ERR_NO_ERROR, broker_delay_ms, + RD_KAFKA_RESP_ERR_NO_ERROR, broker_delay_ms, + RD_KAFKA_RESP_ERR_NO_ERROR, + broker_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to inject ShareAcknowledge delay"); + + /* Step 1: Acknowledge first 2 messages and commit async */ + TEST_SAY( + "Step 1: Acknowledging messages 0-1 and calling commit_async\n"); + ack_range_logged(rkshare, rkmessages, 0, 2, (int)rcvd); + /* The below call should keep the broker busy */ + rd_kafka_share_commit_async(rkshare); + + /* Step 2: Acknowledge next 4 messages and commit async */ + TEST_SAY( + "Step 2: Acknowledging messages 2-5 and calling commit_async " + "(should cache)\n"); + ack_range_logged(rkshare, rkmessages, 2, 6, (int)rcvd); + rd_kafka_share_commit_async(rkshare); + + /* Step 3: Acknowledge next 4 messages and commit sync */ + TEST_SAY( + "Step 3: Acknowledging messages 6-9 and calling " + "commit_sync (should cache)\n"); + ack_range_logged(rkshare, rkmessages, 6, 10, (int)rcvd); + + rd_kafka_share_commit_sync(rkshare, 1000, &commit_result); + TEST_ASSERT(commit_result != NULL, + "Expected non-NULL commit_sync result"); + TEST_SAY("commit_sync returned %d partition result(s)\n", + commit_result->cnt); + for (i = 0; i < commit_result->cnt; i++) { + rd_kafka_topic_partition_t *p = &commit_result->elems[i]; + TEST_SAY(" result[%d] %s [%" PRId32 "] err=%s\n", i, p->topic, + p->partition, rd_kafka_err2str(p->err)); + TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT, + "Expected REQUEST_TIMED_OUT for %s [%" PRId32 + "], got %s", + p->topic, p->partition, rd_kafka_err2str(p->err)); + } + rd_kafka_topic_partition_list_destroy(commit_result); + + /* Destroy all consumed messages */ + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + TEST_SAY("Calling destroy with flags 0x%x\n", destroy_flags); + t_start = test_clock(); + destroy_share_consumer(rkshare, destroy_flags); + t_elapsed_ms = (test_clock() - t_start) / 1000; + TEST_SAY("Destroy completed in %" PRId64 " ms\n", t_elapsed_ms); + + if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) { + /* NO_CONSUMER_CLOSE: destroy must NOT wait for the + * delayed broker. */ + TEST_ASSERT(t_elapsed_ms < 2000, + "Destroy(NO_CONSUMER_CLOSE) took %" PRId64 + " ms, expected < 2000 ms", + t_elapsed_ms); + verify_ack_receipts(&receipts, NULL, 0, + "cached-acks NO_CONSUMER_CLOSE"); + } else { + /* Full close: destroy waits for the existing commit + * async request and the session-leave request. Expect + * roughly 2 * broker_delay_ms plus small overhead. */ + int64_t expected_max_ms = 2 * broker_delay_ms + 2000; + TEST_ASSERT(t_elapsed_ms <= expected_max_ms, + "Destroy took %" PRId64 " ms, expected <= %" PRId64 + " ms", + t_elapsed_ms, expected_max_ms); + expected_ack_t expected[] = { + {topic, 0, RD_KAFKA_RESP_ERR_NO_ERROR, 10}, + }; + verify_ack_receipts(&receipts, expected, 1, + "cached-acks full close"); + } + + ack_receipts_destroy(&receipts); + + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +/** + * @brief Stage a deterministic broker decommission so the next request + * sent to \p target_broker_id (on ApiKey \p blocked_api_key) is + * stamped with __DESTROY_BROKER. + * + * Caller invariants when calling this: + * - Cluster has at least target_broker_id and surviving_broker_id. + * - target_partition's leader in the mock is target_broker_id. + * - No \p blocked_api_key request is currently in flight to + * target_broker_id (the injected delay is consumed by the + * NEXT one). + * + * After this returns: + * - Mock state has target_partition's leader migrated to + * surviving_broker_id and target_broker_id removed from + * metadata. + * - A 2s delay is queued on the next MetadataResponse from + * BOTH brokers — the client has NOT yet observed the + * topology change. + * - A long (30s) delay is queued on the next + * blocked_api_key response from target_broker_id. + * - We've slept past one refresh tick, so the periodic + * MetadataRequest is in flight (held by the mock). + * + * Now when the caller invokes the action under test (which + * will ship a blocked_api_key request to target_broker_id), + * the timeline is: + * - The request lands on target_broker_id and parks (30s). + * - The held MetadataResponse fires (2s in), the client + * sees target_broker_id is gone, runs + * rd_kafka_broker_decommission(target_broker_id). + * - target_broker_id's termination path runs + * bufq_timeout_scan over its outbufs/waitresps and stamps + * the parked request with __DESTROY_BROKER. + * + * Implementation detail: the Metadata RTT injection only takes + * effect because of the rd_kafka_mock_next_request_error() call + * we added to the Metadata handler in rdkafka_mock_handlers.c. + * Without that call, the mock ignores Metadata RTT injections. + */ +static void stage_broker_decommission(rd_kafka_mock_cluster_t *mcluster, + const char *topic, + int32_t target_partition, + int32_t target_broker_id, + int32_t surviving_broker_id, + int16_t blocked_api_key) { + const int metadata_delay_ms = 2000; + const int blocked_api_delay_ms = 30000; + const int refresh_settle_ms = 700; /* > 500ms refresh interval */ + + TEST_SAY("Staging decommission: target broker=%" PRId32 + ", surviving broker=%" PRId32 ", target partition=%" PRId32 + ", blocked ApiKey=%hd\n", + target_broker_id, surviving_broker_id, target_partition, + blocked_api_key); + + /* Delay metadata responses from both brokers so the client + * doesn't observe the topology change until AFTER the caller's + * action has shipped its request to the target broker. */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, target_broker_id, RD_KAFKAP_Metadata, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, metadata_delay_ms); + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, surviving_broker_id, RD_KAFKAP_Metadata, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, metadata_delay_ms); + + /* Delay the blocked-API response on the target broker so the + * caller's request stays parked until decommission fires. */ + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, target_broker_id, blocked_api_key, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, blocked_api_delay_ms); + + /* Migrate the target partition's leader, then remove the + * target broker from metadata. Target broker's TCP connections + * stay alive; in-flight requests remain parked. */ + TEST_ASSERT(rd_kafka_mock_partition_set_leader( + mcluster, topic, target_partition, + surviving_broker_id) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to migrate partition %" PRId32 " leader", + target_partition); + TEST_ASSERT(rd_kafka_mock_broker_remove_from_metadata( + mcluster, target_broker_id) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to remove broker %" PRId32 " from metadata", + target_broker_id); + + /* Sleep past one refresh tick so the next periodic refresh + * fires (and parks behind the metadata delay we just injected). */ + rd_usleep(refresh_settle_ms * 1000, NULL); +} + +/** + * @brief Test that commit_sync returns __DESTROY_BROKER for partitions + * whose broker is decommissioned mid-flight. + * + * Setup: 2-broker mock cluster, 2-partition topic with p0 led + * by broker 1 and p1 led by broker 2. + * + * Flow: + * 1. Drain both partitions. The last batch's partition tells + * us which broker is currently holding unacked records — + * that's the broker we'll decommission. + * 2. Inject a 2s delay on the target broker's NEXT + * MetadataResponse. + * 3. Synchronously update mock state: migrate target + * partition's leader to the surviving broker, and remove + * the target broker from metadata. Connection stays alive. + * 4. Sleep > topic.metadata.refresh.interval.ms (500ms) so the + * periodic refresh tick fires and queues a MetadataRequest + * on some broker. If it lands on the target broker, the + * request sits in front of any subsequent ShareAck on the + * same connection (response is held 2s by the injection). + * If it lands on the surviving broker, the response comes + * back quickly. Either way, the parsed response sees the + * target broker missing and fires + * rd_kafka_broker_decommission() against it. + * 5. commit_sync ships the unacked batch's ShareAck to the + * target broker. The decommission's OP_TERMINATE handler + * stamps the in-flight ShareAck with __DESTROY_BROKER. + * + * Assertion: commit_sync result has 1 entry for the target + * partition with err == __DESTROY_BROKER. + */ +static void test_broker_decommission_with_commit_sync(int destroy_flags, + rd_bool_t explicit_ack) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + rd_kafka_topic_partition_list_t *result = NULL; + const char *topic; + const char *group = "0179-decommission-commit-sync"; + int32_t target_broker_id = -1; + int32_t surviving_broker_id = -1; + int32_t target_partition = -1; + size_t rcvd = 0; + int attempts = 0; + int i; + + SUB_TEST_QUICK("destroy_flags=0x%x ack_mode=%s", destroy_flags, + explicit_ack ? "explicit" : "implicit"); + + /* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that + * fire when the target broker connection is dropped. */ + test_curr->is_fatal_cb = decommission_is_fatal_cb; + + /* 2-broker mock cluster */ + mcluster = test_mock_cluster_new(2, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + /* 2-partition topic: p0 led by broker 1, p1 led by broker 2. + * Consuming + acking from both partitions establishes UP + * connections to both brokers. */ + topic = test_mk_topic_name("0179-decommission-commit-sync", 1); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 0 leader to broker 1"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 1 leader to broker 2"); + + /* Produce TEST_MSGS/2 messages to each partition. */ + TEST_SAY("Producing %d messages to each partition of topic %s\n", + TEST_MSGS / 2, topic); + test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + + TEST_SAY("Creating share consumer (%s ack)\n", + explicit_ack ? "explicit" : "implicit"); + rkshare = new_share_consumer_for_mock_test(bootstraps, group, + explicit_ack, NULL); + subscribe_topics(rkshare, &topic, 1); + + TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS, + MAX_CONSUME_ATTEMPTS); + while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) { + size_t batch_rcvd; + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + attempts++; + continue; + } + + batch_rcvd = rd_kafka_messages_count(rkmessages); + if (batch_rcvd > 0) { + TEST_SAY("Attempt %d: consumed %d messages\n", attempts, + (int)batch_rcvd); + rcvd += batch_rcvd; + + if (explicit_ack) { + size_t k; + for (k = 0; k < batch_rcvd; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(rkmessages, + k); + if (rkm) + rd_kafka_share_acknowledge( + rkshare, rkm); + } + } + } + attempts++; + } + TEST_ASSERT(rcvd == TEST_MSGS, + "Expected %d messages, got %d after %d attempts", TEST_MSGS, + (int)rcvd, attempts); + TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd, + attempts); + + /* The last message of the last batch tells us which broker + * holds the unacked records that commit_sync will ship to. */ + { + size_t _last_cnt = rd_kafka_messages_count(rkmessages); + rd_kafka_message_t *_last_rkm = + _last_cnt > 0 + ? rd_kafka_messages_get(rkmessages, _last_cnt - 1) + : NULL; + TEST_ASSERT(_last_rkm != NULL, + "Expected last batch to be non-empty"); + target_partition = _last_rkm->partition; + } + target_broker_id = (target_partition == 0) ? 1 : 2; + surviving_broker_id = (target_broker_id == 1) ? 2 : 1; + TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32 + ", surviving broker = %" PRId32 "\n", + target_partition, target_broker_id, surviving_broker_id); + + stage_broker_decommission(mcluster, topic, target_partition, + target_broker_id, surviving_broker_id, + RD_KAFKAP_ShareAcknowledge); + + TEST_SAY("Calling commit_sync (timeout 60s)\n"); + error = rd_kafka_share_commit_sync(rkshare, 60000, &result); + + if (error) { + TEST_SAY("commit_sync returned error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } + + TEST_ASSERT(result != NULL, + "Expected commit_sync to return a non-NULL results list"); + TEST_SAY("commit_sync returned %d partition result(s)\n", result->cnt); + + /* Only the last batch's acks reach commit_sync, so we expect + * exactly one partition result: the target partition. */ + TEST_ASSERT(result->cnt == 1, + "Expected 1 partition result (partition %" PRId32 + " only), got %d", + target_partition, result->cnt); + + for (i = 0; i < result->cnt; i++) { + rd_kafka_topic_partition_t *p = &result->elems[i]; + TEST_SAY(" result[%d] %s [%" PRId32 "] err=%s\n", i, p->topic, + p->partition, rd_kafka_err2str(p->err)); + TEST_ASSERT(p->partition == target_partition, + "Expected partition %" PRId32 + " in result, got %" PRId32, + target_partition, p->partition); + /* __DESTROY_BROKER from the broker-thread decommission + * path is translated to __TRANSPORT at the app-facing + * funnel (rd_kafka_share_commit_sync_apply_result), to + * match the Java NetworkException/DisconnectException + * terminal surface. */ + TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR__TRANSPORT, + "Expected __TRANSPORT for partition %" PRId32 + " (broker %" PRId32 ", decommissioned), got %s", + target_partition, target_broker_id, + rd_kafka_err2str(p->err)); + } + + rd_kafka_topic_partition_list_destroy(result); + + /* Cleanup */ + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + destroy_share_consumer(rkshare, destroy_flags); + test_mock_cluster_destroy(mcluster); + + /* Restore the default fatal-error handler. */ + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that a broker decommissioned while a piggybacked + * ShareAck is in flight is handled gracefully: no app-visible + * error surfaces from consume_batch. + * + * Setup mirrors test_broker_decommission_with_commit_sync — + * 2-broker mock cluster, 2-partition topic with RF=2, drain + * both partitions in implicit-ack mode. The last batch's + * records stay ACQUIRED on whichever broker served them + * (target broker, picked dynamically from the last batch's + * partition). The next consume_batch piggybacks an implicit + * ShareAck onto a ShareFetch sent to the target broker — that + * request is what we want stamped __DESTROY_BROKER. + * + * After staging the decommission, we call consume_batch again. + * The fanout sends ShareFetch (with piggybacked ack) to the + * target broker — parked behind the injected ShareAck delay. + * When the metadata response triggers the decommission, the + * parked request is stamped __DESTROY_BROKER via + * bufq_timeout_scan. consume_batch returns no error and no + * messages. + */ +static void test_broker_decommission_with_consume_batch(int destroy_flags) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + const char *topic; + const char *group = "0179-decommission-consume-batch"; + int32_t target_broker_id = -1; + int32_t surviving_broker_id = -1; + int32_t target_partition = -1; + size_t rcvd = 0; + size_t fetch_rcvd = 0; + int attempts = 0; + int32_t surviving_part; + expected_ack_t expected[2]; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + SUB_TEST_QUICK(); + + /* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that + * fire when the target broker connection is dropped. */ + test_curr->is_fatal_cb = decommission_is_fatal_cb; + + /* 2-broker mock cluster */ + mcluster = test_mock_cluster_new(2, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + /* 2-partition topic, RF=2: p0 led by broker 1, p1 led by broker 2. */ + topic = test_mk_topic_name("0179-decommission-consume-batch", 1); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 0 leader to broker 1"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 1 leader to broker 2"); + + /* Produce TEST_MSGS/2 messages to each partition. */ + TEST_SAY("Producing %d messages to each partition of topic %s\n", + TEST_MSGS / 2, topic); + test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + + TEST_SAY("Creating share consumer (implicit ack) with ack_cb\n"); + rkshare = new_share_consumer_for_mock_test(bootstraps, group, rd_false, + &receipts); + subscribe_topics(rkshare, &topic, 1); + + /* Drain both partitions in implicit-ack mode. No trailing-empty + * flush — the last batch's records stay ACQUIRED on whichever + * broker served them. */ + TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS, + MAX_CONSUME_ATTEMPTS); + while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) { + size_t batch_rcvd; + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + attempts++; + continue; + } + + batch_rcvd = rd_kafka_messages_count(rkmessages); + if (batch_rcvd > 0) { + TEST_SAY("Attempt %d: consumed %d messages\n", attempts, + (int)batch_rcvd); + rcvd += batch_rcvd; + } + attempts++; + } + TEST_ASSERT(rcvd == TEST_MSGS, + "Expected %d messages, got %d after %d attempts", TEST_MSGS, + (int)rcvd, attempts); + TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd, + attempts); + + /* The last message of the last batch tells us which broker + * holds the unacked records — that's the broker the next + * consume_batch's piggybacked ShareAck will target. */ + { + size_t _last_cnt = rd_kafka_messages_count(rkmessages); + rd_kafka_message_t *_last_rkm = + _last_cnt > 0 + ? rd_kafka_messages_get(rkmessages, _last_cnt - 1) + : NULL; + TEST_ASSERT(_last_rkm != NULL, + "Expected last batch to be non-empty"); + target_partition = _last_rkm->partition; + } + target_broker_id = (target_partition == 0) ? 1 : 2; + surviving_broker_id = (target_broker_id == 1) ? 2 : 1; + TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32 + ", surviving broker = %" PRId32 "\n", + target_partition, target_broker_id, surviving_broker_id); + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + rcvd = 0; + + rd_kafka_mock_broker_push_request_error_rtts( + mcluster, target_broker_id, RD_KAFKAP_ShareFetch, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, 30000); + + stage_broker_decommission(mcluster, topic, target_partition, + target_broker_id, surviving_broker_id, + RD_KAFKAP_ShareAcknowledge); + + /* Call consume_batch — whichever request lands on the target + * broker (ShareFetch with piggybacked ack OR standalone + * ShareAcknowledge) parks behind its injected delay; when the + * metadata response triggers the decommission, the parked + * request is stamped __DESTROY_BROKER. The surviving broker + * has no new messages, so consume_batch times out cleanly + * with 0 messages and no app-visible error. */ + TEST_SAY( + "Calling consume_batch — expecting no messages and no " + "app-visible error\n"); + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + fetch_rcvd = rd_kafka_messages_count(rkmessages); + + TEST_ASSERT(error == NULL, + "Expected consume_batch to return NULL error, got %s", + error ? rd_kafka_error_string(error) : "(null)"); + TEST_ASSERT(fetch_rcvd == 0, + "Expected 0 messages from consume_batch (target broker " + "decommissioned, surviving broker has no new " + "messages), got %d", + (int)fetch_rcvd); + + drain_ack_callbacks(rkshare); + + /* Assert ack callback receipts. + * + * Drain loop: each batch from the SURVIVING partition gets + * implicitly acked when the next ShareFetch on the surviving + * broker piggybacks the ack — those records appear in the + * callback with NO_ERROR. The TARGET partition's last batch + * stays ACQUIRED (no further piggyback happens on that + * broker before we stage the decommission). + * + * Second consume_batch: the piggybacked ShareAck for the + * target partition's ACQUIRED batch parks on the target + * broker and gets stamped __DESTROY_BROKER when the + * decommission fires; __DESTROY_BROKER is translated to + * __TRANSPORT at the app-facing funnel. + * + * surviving_partition: TEST_MSGS/2 offsets, NO_ERROR + * target_partition: TEST_MSGS/2 offsets, __TRANSPORT */ + surviving_part = (target_partition == 0) ? 1 : 0; + expected[0] = (expected_ack_t) { + topic, surviving_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2}; + expected[1] = + (expected_ack_t) {topic, target_partition, + RD_KAFKA_RESP_ERR__TRANSPORT, TEST_MSGS / 2}; + verify_ack_receipts(&receipts, expected, 2, "consume_batch"); + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + destroy_share_consumer(rkshare, destroy_flags); + test_mock_cluster_destroy(mcluster); + + ack_receipts_destroy(&receipts); + + /* Restore the default fatal-error handler. */ + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that rd_kafka_share_consumer_close() completes gracefully + * when one of the brokers it ships acks/leaves to is decommissioned + * mid-call. + * + * Setup mirrors test_broker_decommission_with_commit_sync — + * 2-broker mock cluster, 2-partition topic with RF=2, fast + * metadata refresh. close() will then ship a ShareAck + * for those records to whichever broker served them, + * plus session-leave requests to both brokers. + * + * The target broker (the one holding the unacked batch) gets a + * 5s ShareAck rtt delay. While that ShareAck is parked, the + * background thread removes the broker from mock metadata; the + * client decommissions it; the in-flight ShareAck is purged + * with __DESTROY_BROKER. + * + * Expectation: rd_kafka_share_consumer_close() returns NULL (no + * error) and does not hang. destroy() afterwards also completes. + */ +static void test_broker_decommission_during_close(int destroy_flags, + rd_bool_t explicit_ack) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + const char *topic; + const char *group = "0179-decommission-close"; + int32_t target_broker_id = -1; + int32_t surviving_broker_id = -1; + int32_t target_partition = -1; + size_t rcvd = 0; + int attempts = 0; + int32_t surviving_part; + expected_ack_t expected[2]; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + SUB_TEST_QUICK("destroy_flags=0x%x ack_mode=%s", destroy_flags, + explicit_ack ? "explicit" : "implicit"); + + /* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that + * fire when the target broker connection is dropped. */ + test_curr->is_fatal_cb = decommission_is_fatal_cb; + + /* 2-broker mock cluster */ + mcluster = test_mock_cluster_new(2, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + /* 2-partition topic, RF=2: p0 led by broker 1, p1 led by broker 2. */ + topic = test_mk_topic_name("0179-decommission-close", 1); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 0 leader to broker 1"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 1 leader to broker 2"); + + /* Produce TEST_MSGS/2 messages to each partition. */ + TEST_SAY("Producing %d messages to each partition of topic %s\n", + TEST_MSGS / 2, topic); + test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + + TEST_SAY("Creating share consumer (%s ack)\n", + explicit_ack ? "explicit" : "implicit"); + rkshare = new_share_consumer_for_mock_test(bootstraps, group, + explicit_ack, &receipts); + subscribe_topics(rkshare, &topic, 1); + + /* Drain both partitions WITHOUT a trailing empty flush — the + * last batch's records remain ACQUIRED (unacked). close() will + * ship a ShareAck for those records to whichever broker served + * them. */ + TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS, + MAX_CONSUME_ATTEMPTS); + while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) { + size_t batch_rcvd; + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + attempts++; + continue; + } + + batch_rcvd = rd_kafka_messages_count(rkmessages); + if (batch_rcvd > 0) { + TEST_SAY("Attempt %d: consumed %d messages\n", attempts, + (int)batch_rcvd); + rcvd += batch_rcvd; + + if (explicit_ack) { + size_t k; + for (k = 0; k < batch_rcvd; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(rkmessages, + k); + if (rkm) + rd_kafka_share_acknowledge( + rkshare, rkm); + } + } + } + attempts++; + } + TEST_ASSERT(rcvd == TEST_MSGS, + "Expected %d messages, got %d after %d attempts", TEST_MSGS, + (int)rcvd, attempts); + TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd, + attempts); + + drain_ack_callbacks(rkshare); + /* The last message of the last batch tells us which broker + * holds the unacked records — that's the broker close()'s + * ShareAck will target. Decommission that broker; the other + * survives. */ + { + size_t _last_cnt = rd_kafka_messages_count(rkmessages); + rd_kafka_message_t *_last_rkm = + _last_cnt > 0 + ? rd_kafka_messages_get(rkmessages, _last_cnt - 1) + : NULL; + TEST_ASSERT(_last_rkm != NULL, + "Expected last batch to be non-empty"); + target_partition = _last_rkm->partition; + } + target_broker_id = (target_partition == 0) ? 1 : 2; + surviving_broker_id = (target_broker_id == 1) ? 2 : 1; + TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32 + ", surviving broker = %" PRId32 "\n", + target_partition, target_broker_id, surviving_broker_id); + + stage_broker_decommission(mcluster, topic, target_partition, + target_broker_id, surviving_broker_id, + RD_KAFKAP_ShareAcknowledge); + + /* Call close(). It ships a ShareAck session leave + * request for the unacked batch to the target broker. + * The target broker's ShareAck is stamped __DESTROY_BROKER + * when the decommission fires; close() must not surface this to the + * app. */ + TEST_SAY( + "Calling rd_kafka_share_consumer_close() — expecting " + "NULL error\n"); + error = rd_kafka_share_consumer_close(rkshare); + + TEST_ASSERT(error == NULL, + "Expected close() to return NULL error, got %s", + error ? rd_kafka_error_string(error) : "(null)"); + + /* Assert ack callback receipts. + * + * The first batch on each broker was piggyback-acked on the + * next ShareFetch during drain — those records appear in the + * callback with NO_ERROR. The LAST batch on each broker is + * still ACQUIRED at close time; close ships those acks + * differently in implicit vs explicit mode: + * + * Implicit ack: the close path only flushes already-converted + * (non-ACQUIRED) entries. ACQUIRED records on the last batch + * are silently dropped — no ShareAck for them goes on the + * wire, so no callback fires for target_partition's last + * batch (which would have been stamped __DESTROY_BROKER). + * Expectation: only surviving_partition NO_ERROR. + * + * Explicit ack: the drain loop explicitly acked every record. + * Those explicit acks are queued and shipped during close. + * The target broker's ack request parks behind the injected + * delay; when the decommission fires, the parked request is + * stamped __DESTROY_BROKER, which the app-facing funnel + * translates to __TRANSPORT before the callback fires. + * surviving_partition records that weren't piggyback-acked + * during drain are also shipped at close and succeed with + * NO_ERROR. + */ + surviving_part = (target_partition == 0) ? 1 : 0; + expected[0] = (expected_ack_t) { + topic, surviving_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2}; + if (explicit_ack) { + expected[1] = (expected_ack_t) {topic, target_partition, + RD_KAFKA_RESP_ERR__TRANSPORT, + TEST_MSGS / 2}; + verify_ack_receipts(&receipts, expected, 2, "close explicit"); + } else { + verify_ack_receipts(&receipts, expected, 1, "close implicit"); + } + + /* Cleanup */ + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + destroy_share_consumer(rkshare, destroy_flags); + test_mock_cluster_destroy(mcluster); + + ack_receipts_destroy(&receipts); + + /* Restore the default fatal-error handler. */ + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + + +/** + * @brief Test that rd_kafka_share_commit_async() does not hang when the + * broker it ships acks to is decommissioned mid-flight. + * + * Setup mirrors test_broker_decommission_during_close — drain both + * partitions without a trailing flush so the last batch's records + * stay ACQUIRED. commit_async() then ships a ShareAck for those + * records to whichever broker served them. The target broker has a + * 5s ShareAck rtt delay; while the request is parked, the + * background thread removes the broker from mock metadata and the + * client decommissions it, purging the in-flight ShareAck with + * __DESTROY_BROKER. + * + * Expectation: rd_kafka_share_commit_async() returns NULL (no + * error) and does not block on the in-flight ack. + * destroy() afterwards completes cleanly. + */ +static void test_broker_decommission_with_commit_async(int destroy_flags, + rd_bool_t explicit_ack) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + const char *topic; + const char *group = "0179-decommission-commit-async"; + int32_t target_broker_id = -1; + int32_t surviving_broker_id = -1; + int32_t target_partition = -1; + size_t rcvd = 0; + int attempts = 0; + int32_t surviving_part; + expected_ack_t expected[2]; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + SUB_TEST_QUICK("destroy_flags=0x%x ack_mode=%s", destroy_flags, + explicit_ack ? "explicit" : "implicit"); + + /* Suppress expected __TRANSPORT / __ALL_BROKERS_DOWN errors that + * fire when the target broker connection is dropped. */ + test_curr->is_fatal_cb = decommission_is_fatal_cb; + + /* 2-broker mock cluster */ + mcluster = test_mock_cluster_new(2, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + /* 2-partition topic, RF=2: p0 led by broker 1, p1 led by broker 2. */ + topic = test_mk_topic_name("0179-decommission-commit-async", 1); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 2, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 0 leader to broker 1"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 1, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set partition 1 leader to broker 2"); + + /* Produce TEST_MSGS/2 messages to each partition. */ + TEST_SAY("Producing %d messages to each partition of topic %s\n", + TEST_MSGS / 2, topic); + test_produce_msgs_easy_v(topic, 0, 0, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + test_produce_msgs_easy_v(topic, 0, 1, 0, TEST_MSGS / 2, 16, + "bootstrap.servers", bootstraps, NULL); + + /* Implicit-ack share consumer with fast metadata refresh so the + * client picks up the target broker's removal during the 5s + * ShareAck delay. */ + TEST_SAY("Creating share consumer (%s ack) with ack_cb\n", + explicit_ack ? "explicit" : "implicit"); + rkshare = new_share_consumer_for_mock_test(bootstraps, group, + explicit_ack, &receipts); + subscribe_topics(rkshare, &topic, 1); + + /* Drain both partitions WITHOUT a trailing empty flush — the + * last batch's records remain ACQUIRED (unacked). + * commit_async() will ship a ShareAck for those records to + * whichever broker served them. */ + TEST_SAY("Consuming up to %d messages (max %d attempts)\n", TEST_MSGS, + MAX_CONSUME_ATTEMPTS); + while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) { + size_t batch_rcvd; + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + attempts++; + continue; + } + + batch_rcvd = rd_kafka_messages_count(rkmessages); + if (batch_rcvd > 0) { + TEST_SAY("Attempt %d: consumed %d messages\n", attempts, + (int)batch_rcvd); + rcvd += batch_rcvd; + + if (explicit_ack) { + size_t k; + for (k = 0; k < batch_rcvd; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(rkmessages, + k); + if (rkm) + rd_kafka_share_acknowledge( + rkshare, rkm); + } + } + } + attempts++; + } + TEST_ASSERT(rcvd == TEST_MSGS, + "Expected %d messages, got %d after %d attempts", TEST_MSGS, + (int)rcvd, attempts); + TEST_SAY("Consumed %d msgs total in %d attempts\n", (int)rcvd, + attempts); + + drain_ack_callbacks(rkshare); + /* The last message of the last batch tells us which broker + * holds the unacked records — that's the broker the + * commit_async ShareAck will target. Decommission that broker; + * the other survives. */ + { + size_t _last_cnt = rd_kafka_messages_count(rkmessages); + rd_kafka_message_t *_last_rkm = + _last_cnt > 0 + ? rd_kafka_messages_get(rkmessages, _last_cnt - 1) + : NULL; + TEST_ASSERT(_last_rkm != NULL, + "Expected last batch to be non-empty"); + target_partition = _last_rkm->partition; + } + target_broker_id = (target_partition == 0) ? 1 : 2; + surviving_broker_id = (target_broker_id == 1) ? 2 : 1; + TEST_SAY("Target partition = %" PRId32 ", target broker = %" PRId32 + ", surviving broker = %" PRId32 "\n", + target_partition, target_broker_id, surviving_broker_id); + + stage_broker_decommission(mcluster, topic, target_partition, + target_broker_id, surviving_broker_id, + RD_KAFKAP_ShareAcknowledge); + + /* Call commit_async(). It should return immediately without + * blocking on the in-flight ack. The parked ShareAck on the + * target broker will be stamped __DESTROY_BROKER when the + * decommission fires shortly after. */ + TEST_SAY( + "Calling rd_kafka_share_commit_async() — expecting " + "NULL error and no hang\n"); + error = rd_kafka_share_commit_async(rkshare); + + TEST_ASSERT(error == NULL, + "Expected commit_async() to return NULL error, got %s", + error ? rd_kafka_error_string(error) : "(null)"); + + /* Cleanup messages before destroy. */ + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + destroy_share_consumer(rkshare, destroy_flags); + test_mock_cluster_destroy(mcluster); + + /* Assert ack callback receipts. + * + * Full close (destroy_flags=0): destroy waits for in-flight + * requests. The metadata-driven decommission fires within ~2s, + * stamping the parked ShareAck with __DESTROY_BROKER. + * + * NO_CONSUMER_CLOSE (destroy_flags=0x8): destroy short- + * circuits without waiting for the parked ShareAck. No + * callback fires for the target partition's records. Only the + * surviving partition's ack — which completed before — + * surfaces. */ + surviving_part = (target_partition == 0) ? 1 : 0; + expected[0] = (expected_ack_t) { + topic, surviving_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2}; + if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) { + verify_ack_receipts(&receipts, expected, 1, + "commit_async NO_CONSUMER_CLOSE"); + } else { + /* __DESTROY_BROKER stamped on parked acks is translated + * to __TRANSPORT at the app-facing funnel. */ + expected[1] = (expected_ack_t) {topic, target_partition, + RD_KAFKA_RESP_ERR__TRANSPORT, + TEST_MSGS / 2}; + verify_ack_receipts(&receipts, expected, 2, + "commit_async full close"); + } + + ack_receipts_destroy(&receipts); + + /* Restore the default fatal-error handler. */ + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + +static void test_leader_migration_mid_session_destroy(int destroy_flags) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + const char *topic = "0179-leader-migration-mid-session"; + const char *group = "0179-leader-migration-mid-session"; + const int msgs_per_round = 5; + size_t rcvd = 0; + int attempts = 0; + + SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags); + + mcluster = test_mock_cluster_new(2, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + /* 1 partition, RF=2 so both brokers know about it. Initial + * leader = broker 1. */ + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to set initial leader to broker 1"); + + /* Produce a first batch so the consumer can fetch from broker 1. */ + TEST_SAY("Producing %d messages to broker 1\n", msgs_per_round); + test_produce_msgs_easy_v(topic, 0, 0, 0, msgs_per_round, 16, + "bootstrap.servers", bootstraps, NULL); + + rkshare = new_share_consumer_for_mock_test( + bootstraps, group, rd_false /* implicit */, NULL); + subscribe_topics(rkshare, &topic, 1); + + /* Round 1: consume from broker 1. After this, the toppar is in + * broker 1's toppars_in_session. */ + TEST_SAY("Round 1: consume from broker 1 (initial leader)\n"); + while (rcvd < (size_t)msgs_per_round && + attempts++ < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + if (error) + rd_kafka_error_destroy(error); + else + rcvd += rd_kafka_messages_count(rkmessages); + } + TEST_ASSERT(rcvd >= (size_t)msgs_per_round, + "Round 1: expected %d msgs, got %d", msgs_per_round, + (int)rcvd); + TEST_SAY("Round 1: consumed %d messages\n", (int)rcvd); + + /* Migrate the partition leader to broker 2. */ + TEST_SAY("Migrating partition leader from broker 1 to broker 2\n"); + TEST_ASSERT(rd_kafka_mock_partition_set_leader(mcluster, topic, 0, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to migrate leader to broker 2"); + + /* Produce a second batch so the consumer has something to fetch + * from the new leader. */ + TEST_SAY("Producing %d more messages (now under broker 2)\n", + msgs_per_round); + test_produce_msgs_easy_v(topic, 0, 0, 0, msgs_per_round, 16, + "bootstrap.servers", bootstraps, NULL); + + /* Round 2: consume from broker 2. The first ShareFetch reply + * from broker 2 will add the toppar into broker + * 2's toppars_in_session. With the fix, broker 1's list + * is cleared by PARTITION_LEAVE before this happens. */ + TEST_SAY("Round 2: consume from broker 2 (new leader)\n"); + attempts = 0; + while (rcvd < (size_t)(2 * msgs_per_round) && + attempts++ < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + if (error) + rd_kafka_error_destroy(error); + else + rcvd += rd_kafka_messages_count(rkmessages); + } + TEST_ASSERT(rcvd >= (size_t)(2 * msgs_per_round), + "Round 2: expected %d total msgs, got %d", + 2 * msgs_per_round, (int)rcvd); + TEST_SAY("Round 2: consumed %d total messages\n", (int)rcvd); + + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + /* Close + destroy. Without the leader-migration fix, broker 1's + * destroy_final would assert on non-empty toppars_in_session. */ + TEST_SAY("Calling destroy (flags 0x%x)\n", destroy_flags); + destroy_share_consumer(rkshare, destroy_flags); + + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +/** + * @brief Destroy while the consumer's cgrp is mid-rebalance. + * + * Drives the cgrp into an intermediate join-state by: + * 1. Subscribing and consuming at least one batch (cgrp now in steady + * state with a live assignment). + * 2. Injecting a long RTT delay on ShareGroupHeartbeat so the next + * heartbeat sent by the cgrp will be stuck in-flight. + * 3. Calling unsubscribe, which drives the cgrp into + * wait-unassign-call -> wait-incr-unassign-to-complete and ends up + * waiting on the (now-blocked) leave heartbeat. + * 4. Calling destroy with the given flags. Destroy must NOT wait the + * full injected RTT. + */ +static void test_destroy_during_rebalance(int destroy_flags) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *rkmessages = NULL; + const char *topic = "0179-destroy-during-rebalance"; + const char *group = "0179-destroy-during-rebalance"; + const int heartbeat_delay_ms = 10000; + size_t rcvd = 0; + int attempts = 0; + rd_ts_t t_start, t_elapsed_ms; + + SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + /* Produce so the consumer's first consume_batch returns a real + * message, guaranteeing the cgrp reached up/steady with a live + * assignment. */ + test_produce_msgs_easy_v(topic, 0, 0, 0, 10, 16, "bootstrap.servers", + bootstraps, NULL); + + rkshare = new_share_consumer_for_mock_test( + bootstraps, group, rd_false /* implicit ack */, NULL); + subscribe_topics(rkshare, &topic, 1); + + TEST_SAY("Waiting for first batch (signals assignment is live)\n"); + while (rcvd == 0 && attempts++ < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + if (error) + rd_kafka_error_destroy(error); + else + rcvd = rd_kafka_messages_count(rkmessages); + } + TEST_ASSERT(rcvd > 0, + "Expected at least 1 msg before forcing rebalance; " + "got %d after %d attempts", + (int)rcvd, attempts); + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + /* Block all subsequent ShareGroupHeartbeats. The next heartbeat + * sent by the cgrp (including the leave-group heartbeat) will be + * stuck in flight for heartbeat_delay_ms. */ + TEST_SAY("Injecting %dms RTT delay on ShareGroupHeartbeat\n", + heartbeat_delay_ms); + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, + heartbeat_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to inject heartbeat delay"); + + /* Drive the cgrp into the leave-flow. */ + TEST_SAY("Calling unsubscribe to drive cgrp into wait-incr-unassign\n"); + TEST_ASSERT(!rd_kafka_share_unsubscribe(rkshare), "unsubscribe failed"); + + /* Give the cgrp main thread time to process the unsubscribe op + * and transition into the intermediate join-state. */ + rd_sleep(1); + + TEST_SAY("Calling destroy mid-rebalance (flags 0x%x)\n", destroy_flags); + t_start = test_clock(); + destroy_share_consumer(rkshare, destroy_flags); + t_elapsed_ms = (test_clock() - t_start) / 1000; + TEST_SAY("Destroy completed in %" PRId64 " ms\n", t_elapsed_ms); + + if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) { + /* No-close: destroy must skip the cgrp leave flow and + * return promptly, regardless of the heartbeat delay. */ + TEST_ASSERT(t_elapsed_ms < 2000, + "Destroy(NO_CONSUMER_CLOSE) took %" PRId64 + " ms, expected < 2000 ms", + t_elapsed_ms); + } else { + /* Full close: destroy waits for the cgrp to finish its + * leave flow, which is blocked on the heartbeat. It must + * still complete in roughly heartbeat_delay_ms (plus + * small overhead) and never wait indefinitely. */ + TEST_ASSERT(t_elapsed_ms < heartbeat_delay_ms + 5000, + "Destroy took %" PRId64 " ms, expected < %d ms", + t_elapsed_ms, heartbeat_delay_ms + 5000); + } + + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +/** + * @brief Test destroying a share consumer after a fatal error. + * + * Drives the consumer to steady state, acks half the consumed batch, + * then injects a 30s RTT delay on the next ShareAcknowledge so a real + * close path would have to wait that long. Trigger a fatal error + * (which causes destroy_app to promote flags to NO_CONSUMER_CLOSE, + * then call destroy. + * + * Destroy must short-circuit the close path and complete promptly + * regardless of the supplied destroy_flags — the delayed ShareAck + * never gets sent, and broker threads must exit without waiting. + * + * @param destroy_flags Destroy flags (0 or + * RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE). Both + * must complete fast under fatal-error promotion. + */ +static void test_destroy_with_fatal_error(int destroy_flags) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_t *rk; + rd_kafka_error_t *error; + rd_kafka_resp_err_t err; + rd_kafka_messages_t *rkmessages = NULL; + const char *topic = "0179-destroy-fatal-error"; + const char *group = "0179-destroy-fatal-error"; + const int total_msgs = 10; + const int share_ack_delay_ms = 30000; + size_t rcvd = 0; + int attempts = 0; + size_t i; + rd_ts_t t_start, t_elapsed_ms; + + SUB_TEST_QUICK("destroy_flags=0x%x", destroy_flags); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + test_produce_msgs_easy_v(topic, 0, 0, 0, total_msgs, 16, + "bootstrap.servers", bootstraps, NULL); + + rkshare = new_share_consumer_for_mock_test( + bootstraps, group, rd_true /* explicit */, NULL); + rk = test_share_consumer_get_rk(rkshare); + subscribe_topics(rkshare, &topic, 1); + + TEST_SAY("Consuming %d msgs\n", total_msgs); + while (rcvd < (size_t)total_msgs && attempts++ < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &rkmessages); + if (error) + rd_kafka_error_destroy(error); + else + rcvd = rd_kafka_messages_count(rkmessages); + } + TEST_ASSERT(rcvd >= (size_t)total_msgs, + "Expected %d msgs, got %d after %d attempts", total_msgs, + (int)rcvd, attempts); + + TEST_SAY("Acknowledging %d/%d messages (no commit)\n", (int)(rcvd / 2), + (int)rcvd); + for (i = 0; i < rcvd / 2; i++) { + rd_kafka_message_t *rkm = rd_kafka_messages_get(rkmessages, i); + if (rkm) + rd_kafka_share_acknowledge(rkshare, rkm); + } + + /* Inject a long delay on the next ShareAcknowledge so a normal + * close path would block on it. The fatal-error path must NOT + * wait. */ + TEST_SAY("Injecting %dms RTT delay on ShareAcknowledge\n", + share_ack_delay_ms); + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, + share_ack_delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to inject ShareAcknowledge delay"); + + TEST_SAY("Injecting fatal error\n"); + err = rd_kafka_test_fatal_error( + rk, RD_KAFKA_RESP_ERR__FATAL, + "0179: injected fatal error before destroy"); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "test_fatal_error returned %s", rd_kafka_err2name(err)); + + /* Destroy all consumed messages before share_destroy. */ + rd_kafka_messages_destroy(rkmessages); + rkmessages = NULL; + + TEST_SAY("Calling destroy with flags 0x%x (fatal-error path)\n", + destroy_flags); + t_start = test_clock(); + destroy_share_consumer(rkshare, destroy_flags); + t_elapsed_ms = (test_clock() - t_start) / 1000; + TEST_SAY("Destroy completed in %" PRId64 " ms\n", t_elapsed_ms); + + /* Fatal error short-circuits close regardless of supplied flag, + * so destroy must be prompt and never wait the injected RTT. */ + TEST_ASSERT(t_elapsed_ms < 2000, + "Destroy took %" PRId64 " ms, expected < 2000 ms", + t_elapsed_ms); + + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + + +/** + * @brief Test destroying share consumer with explicit-ack mode. + * + * Consumes a batch of messages, explicitly acknowledges some/all of + * them (no commit), then destroys the consumer. Verifies that destroy + * handles pending acks correctly. + * + * @param destroy_flags Destroy flags (0 or + * RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE). + * @param ack_half If true, ack only the first half of the + * consumed batch (the rest are left unacked). If + * false, ack the full batch. + */ +static void do_test_destroy_with_explicit_ack(int destroy_flags, + rd_bool_t ack_half) { + const char *topic; + const char *group = "0179-destroy-explicit-ack"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *first_batch = NULL; + rd_kafka_messages_t *second_batch = NULL; + size_t rcvd = 0; + size_t first_batch_cnt = 0; + int32_t first_batch_part = -1; + int32_t second_batch_part = -1; + int attempts = 0; + int ack_cnt; + expected_ack_t expected[2]; + int second_part_cnt; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + SUB_TEST("destroy_flags=0x%x ack_half=%s", destroy_flags, + ack_half ? "true" : "false"); + + topic = test_mk_topic_name("0179-destroy-explicit-ack", 1); + TEST_SAY("Creating 2-partition topic %s\n", topic); + test_create_topic_wait_exists(common_admin, topic, 2, -1, 60 * 1000); + + TEST_SAY("Producing %d messages to each partition of topic %s\n", + TEST_MSGS / 2, topic); + test_produce_msgs_simple(common_producer, topic, 0, TEST_MSGS / 2); + test_produce_msgs_simple(common_producer, topic, 1, TEST_MSGS / 2); + + test_share_set_auto_offset_reset(group, "earliest"); + + TEST_SAY("Creating share consumer (explicit ack mode) with ack_cb\n"); + rkshare = + new_share_consumer_for_real_test(group, "explicit", &receipts); + subscribe_topics(rkshare, &topic, 1); + + /* Consume + ack loop. Explicit ack mode requires acking the + * prior batch before the next consume_batch (otherwise it + * errors), so we ack inside the loop. + * + * 2-partition setup with TEST_MSGS/2 per partition: each + * non-empty consume_batch returns one partition's full + * payload (TEST_MSGS/2 records). The 1st non-empty batch is + * always acked in full; we break out once we've consumed all + * TEST_MSGS records (i.e., after the 2nd batch arrives — + * which we then ack outside the loop based on ack_half). */ + TEST_SAY("Consuming %d messages (up to %d attempts)\n", TEST_MSGS, + MAX_CONSUME_ATTEMPTS); + while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_t *batch = NULL; + size_t batch_rcvd; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + attempts++; + continue; + } + + batch_rcvd = rd_kafka_messages_count(batch); + if (batch_rcvd > 0) { + TEST_SAY("Attempt %d: consumed %d messages\n", attempts, + (int)batch_rcvd); + if (first_batch == NULL) { + size_t k; + /* First non-empty batch: ack ALL. The + * next consume_batch will piggyback + * this ack on a ShareFetch. */ + first_batch = batch; + first_batch_cnt = batch_rcvd; + for (k = 0; k < batch_rcvd; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, k); + if (rkm) + rd_kafka_share_acknowledge( + rkshare, rkm); + } + } else { + /* Second non-empty batch — keep it for + * out-of-loop acks below. */ + rd_kafka_messages_destroy(second_batch); + second_batch = batch; + } + rcvd += batch_rcvd; + } else { + rd_kafka_messages_destroy(batch); + } + attempts++; + } + + TEST_ASSERT(rcvd == TEST_MSGS, + "Expected %d messages after %d attempts, got %d", TEST_MSGS, + MAX_CONSUME_ATTEMPTS, (int)rcvd); + TEST_ASSERT(first_batch_cnt == TEST_MSGS / 2, + "Expected first batch == TEST_MSGS/2 (%d), got %d", + TEST_MSGS / 2, (int)first_batch_cnt); + TEST_ASSERT(second_batch != NULL, "Expected a non-NULL second batch"); + + drain_ack_callbacks(rkshare); + { + rd_kafka_message_t *first_rkm = + rd_kafka_messages_get(first_batch, 0); + TEST_ASSERT(first_rkm != NULL, + "Expected non-NULL first batch message"); + first_batch_part = first_rkm->partition; + } + second_batch_part = (first_batch_part == 0) ? 1 : 0; + TEST_SAY("Consumed %d messages (first batch on partition %" PRId32 + ", second batch on partition %" PRId32 ")\n", + (int)rcvd, first_batch_part, second_batch_part); + + /* Ack the second batch: + * ack_half=false: ack all TEST_MSGS/2 records. + * ack_half=true: ack TEST_MSGS/4 records (the first half). + * These acks won't be piggyback-flushed close runs — which is + * exactly what destroy_share_consumer does (with destroy_flags=0). + * With destroy_flags=0x8 these acks are dropped. */ + { + int second_to_ack = + ack_half ? (TEST_MSGS / 4) : (TEST_MSGS / 2); + int k; + for (k = 0; k < second_to_ack; k++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(second_batch, k); + if (rkm) + rd_kafka_share_acknowledge(rkshare, rkm); + } + ack_cnt = (int)first_batch_cnt + second_to_ack; + TEST_SAY( + "Acked first batch (%d) + second batch (%d) = %d / " + "%d total\n", + (int)first_batch_cnt, second_to_ack, ack_cnt, (int)rcvd); + } + + /* Destroy all consumed messages (acked and unacked alike). */ + rd_kafka_messages_destroy(first_batch); + first_batch = NULL; + rd_kafka_messages_destroy(second_batch); + second_batch = NULL; + + destroy_share_consumer(rkshare, destroy_flags); + + /* Assert ack callback receipts. + * + * Acks fired: + * - First batch: always all TEST_MSGS/2 records on + * first_batch_part (acked inside the loop, piggybacked on + * the next ShareFetch — delivered before destroy). + * - Second batch: TEST_MSGS/2 (if !ack_half) or TEST_MSGS/4 + * (if ack_half) records on second_batch_part. These acks + * are queued internally and only flushed during close. + * + * destroy_flags=0 (full close): close flushes the second + * batch's acks. Both partitions appear. + * + * destroy_flags=0x8 (NO_CONSUMER_CLOSE): destroy short- + * circuits, so second-batch acks are dropped. Only + * first_batch_part appears with TEST_MSGS/2. */ + expected[0] = (expected_ack_t) { + topic, first_batch_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2}; + if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) { + verify_ack_receipts(&receipts, expected, 1, + "explicit-ack NO_CONSUMER_CLOSE"); + } else { + second_part_cnt = ack_half ? (TEST_MSGS / 4) : (TEST_MSGS / 2); + expected[1] = (expected_ack_t) {topic, second_batch_part, + RD_KAFKA_RESP_ERR_NO_ERROR, + second_part_cnt}; + verify_ack_receipts(&receipts, expected, 2, + "explicit-ack full close"); + } + + ack_receipts_destroy(&receipts); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test destroying share consumer with implicit-ack mode. + * + * Consumes a batch of messages in implicit-ack mode but never makes a + * follow-up poll (which is what would trigger the implicit ack), then + * destroys the consumer directly. Verifies that destroy handles + * un-acked records gracefully. + * + * @param destroy_flags Destroy flags (0 or + * RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE). + */ +static void do_test_destroy_with_implicit_ack(int destroy_flags) { + const char *topic; + const char *group = "0179-destroy-implicit-ack"; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + rd_kafka_messages_t *first_batch = NULL; + rd_kafka_messages_t *second_batch = NULL; + size_t rcvd = 0; + size_t first_batch_cnt = 0; + int32_t first_batch_part = -1; + int32_t second_batch_part = -1; + int attempts = 0; + expected_ack_t expected[1]; + ack_receipts_t receipts; + + ack_receipts_init(&receipts); + + SUB_TEST("destroy_flags=0x%x", destroy_flags); + + topic = test_mk_topic_name("0179-destroy-implicit-ack", 1); + TEST_SAY("Creating 2-partition topic %s\n", topic); + test_create_topic_wait_exists(common_admin, topic, 2, -1, 60 * 1000); + + /* Explicit produce: TEST_MSGS/2 to each partition. */ + TEST_SAY("Producing %d messages to each partition of topic %s\n", + TEST_MSGS / 2, topic); + test_produce_msgs_simple(common_producer, topic, 0, TEST_MSGS / 2); + test_produce_msgs_simple(common_producer, topic, 1, TEST_MSGS / 2); + + test_share_set_auto_offset_reset(group, "earliest"); + + TEST_SAY("Creating share consumer (implicit ack mode) with ack_cb\n"); + rkshare = + new_share_consumer_for_real_test(group, "implicit", &receipts); + subscribe_topics(rkshare, &topic, 1); + + /* Consume loop. Implicit ack mode: records are auto-acked by + * the consumer when the next consume_batch's ShareFetch + * piggybacks them. The 2-partition setup with TEST_MSGS/2 per + * partition means each non-empty consume_batch returns one + * partition's full payload. After the 2nd consume_batch + * returns, we've consumed all TEST_MSGS records and the 1st + * batch's records' implicit-acks have already been + * piggybacked. The 2nd batch's records are still ACQUIRED. */ + TEST_SAY("Consuming %d messages (up to %d attempts)\n", TEST_MSGS, + MAX_CONSUME_ATTEMPTS); + while (rcvd < TEST_MSGS && attempts < MAX_CONSUME_ATTEMPTS) { + rd_kafka_messages_t *batch = NULL; + size_t batch_rcvd; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + + if (error) { + TEST_SAY("Attempt %d: consume error: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + attempts++; + continue; + } + + batch_rcvd = rd_kafka_messages_count(batch); + if (batch_rcvd > 0) { + TEST_SAY("Attempt %d: consumed %d messages\n", attempts, + (int)batch_rcvd); + if (first_batch == NULL) { + first_batch = batch; + first_batch_cnt = batch_rcvd; + } else { + rd_kafka_messages_destroy(second_batch); + second_batch = batch; + } + rcvd += batch_rcvd; + } else { + rd_kafka_messages_destroy(batch); + } + attempts++; + } + + TEST_ASSERT(rcvd == TEST_MSGS, + "Expected %d messages after %d attempts, got %d", TEST_MSGS, + MAX_CONSUME_ATTEMPTS, (int)rcvd); + TEST_ASSERT(first_batch_cnt == TEST_MSGS / 2, + "Expected first batch == TEST_MSGS/2 (%d), got %d", + TEST_MSGS / 2, (int)first_batch_cnt); + { + rd_kafka_message_t *first_rkm = + rd_kafka_messages_get(first_batch, 0); + TEST_ASSERT(first_rkm != NULL, + "Expected non-NULL first batch message"); + first_batch_part = first_rkm->partition; + } + second_batch_part = (first_batch_part == 0) ? 1 : 0; + TEST_SAY("Consumed %d messages (first batch on partition %" PRId32 + ", second batch on partition %" PRId32 ")\n", + (int)rcvd, first_batch_part, second_batch_part); + + /* Drain pending ack callbacks: the first batch's piggybacked + * acks may still be in flight. */ + drain_ack_callbacks(rkshare); + + /* Destroy all consumed messages before share_destroy. */ + rd_kafka_messages_destroy(first_batch); + first_batch = NULL; + rd_kafka_messages_destroy(second_batch); + second_batch = NULL; + + destroy_share_consumer(rkshare, destroy_flags); + + /* Assert ack callback receipts. + * + * In implicit-ack mode, the 2nd batch's records stay ACQUIRED + * at close/destroy time and are NOT shipped by either close + * path (full or NO_CONSUMER_CLOSE) — so they never surface + * in the callback. Only the 1st batch's piggybacked acks + * (flushed via the 2nd consume_batch's ShareFetch) reach the + * callback. Expectation is identical for both destroy_flags + * variants: TEST_MSGS/2 NO_ERROR on first_batch_part only. */ + (void)second_batch_part; + expected[0] = (expected_ack_t) { + topic, first_batch_part, RD_KAFKA_RESP_ERR_NO_ERROR, TEST_MSGS / 2}; + verify_ack_receipts( + &receipts, expected, 1, + (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) + ? "implicit-ack NO_CONSUMER_CLOSE" + : "implicit-ack full close"); + + ack_receipts_destroy(&receipts); + + SUB_TEST_PASS(); +} + + +/** + * @brief Test destroying share consumer after subscribe/unsubscribe. + * + * This test creates a share consumer, optionally subscribes to topics, + * optionally unsubscribes, then destroys it without consuming any messages. + * Tests various combinations similar to 0116-kafkaconsumer_close. + * + * @param do_subscribe Whether to subscribe to topics + * @param do_unsubscribe Whether to unsubscribe before destroy + * @param destroy_flags Destroy flags (0 or + * RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE) + */ +static void do_test_destroy_with_subscribe_unsubscribe(int do_subscribe, + int do_unsubscribe, + int destroy_flags) { + rd_kafka_share_t *consumer; + rd_kafka_topic_partition_list_t *topics; + const char *topic = "0179-test-destroy-sub-unsub"; + rd_kafka_resp_err_t err; + + SUB_TEST("subscribe=%d, unsubscribe=%d, destroy_flags=0x%x", + do_subscribe, do_unsubscribe, destroy_flags); + + TEST_SAY("Creating share consumer\n"); + consumer = test_create_share_consumer("0179-sub-unsub-destroy-test", + "explicit"); + + if (do_subscribe) { + TEST_SAY("Subscribing to topic: %s\n", topic); + topics = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(topics, topic, + RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(consumer, topics); + TEST_ASSERT(!err, "Subscribe failed: %s", + rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(topics); + } + + if (do_unsubscribe) { + TEST_SAY("Unsubscribing from all topics\n"); + err = rd_kafka_share_unsubscribe(consumer); + TEST_ASSERT(!err, "Unsubscribe failed: %s", + rd_kafka_err2str(err)); + } + + destroy_share_consumer(consumer, destroy_flags); + SUB_TEST_PASS(); +} + + +/** + * @brief Destroy a consumer that drove a mixed-ack chaos workload — + * ACCEPT/RELEASE/REJECT round-robin acks, with commit_async every + * 5 acks and commit_sync every 10 acks. destroy runs on a + * watchdog thread so a hung destroy fails the test loudly instead + * of wedging the binary. + */ +static void do_test_destroy_with_mixed_acks_and_commits(int destroy_flags) { + const char *topic; + const char *group = "0179-destroy-mixed"; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *close_err; + thrd_t destroy_thr; + destroy_watchdog_arg_t arg; + ack_receipts_t receipts; + const int produce_cnt = 40; + const int target = 30; + const int DESTROY_DEADLINE_MS = 15000; + rd_ts_t t_start; + int64_t t_close_ms = 0; + int64_t t_destroy_ms; + size_t rcvd; + size_t j; + int acked = 0; + int sync_cnt = 0; + int async_cnt = 0; + int attempts = 0; + int wait_ok; + rd_bool_t do_close = + !(destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + + SUB_TEST("destroy_flags=0x%x", destroy_flags); + + ack_receipts_init(&receipts); + + topic = test_mk_topic_name("0179-destroy-mixed", 1); + test_create_topic_wait_exists(common_admin, topic, 1, -1, 60 * 1000); + test_produce_msgs_simple(common_producer, topic, 0, produce_cnt); + + test_share_set_auto_offset_reset(group, "earliest"); + rkshare = + new_share_consumer_for_real_test(group, "explicit", &receipts); + subscribe_topics(rkshare, &topic, 1); + + /* Drive chaos: round-robin ACCEPT/RELEASE/REJECT acks with + * interleaved commits. Past `target` we drop each batch's + * message handles without acking — close/destroy must cope + * with whatever inflight state that leaves the lib in. */ + while (acked < target && attempts++ < 80) { + rd_kafka_error_t *err; + rd_kafka_messages_destroy(batch); + batch = NULL; + err = rd_kafka_share_poll(rkshare, 3000, &batch); + if (err) { + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_share_AcknowledgeType_t at; + rd_kafka_resp_err_t aerr; + rd_kafka_error_t *cerr; + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + + if (!rkm) + continue; + + if (rkm->err) + continue; + + if (acked >= target) + continue; + + switch (acked % 3) { + case 0: + at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT; + break; + case 1: + at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE; + break; + default: + at = RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT; + break; + } + + aerr = + rd_kafka_share_acknowledge_type(rkshare, rkm, at); + TEST_ASSERT(!aerr, "acknowledge_type(%d) failed: %s", + (int)at, rd_kafka_err2str(aerr)); + acked++; + + if (acked % 5 == 0) { + cerr = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!cerr, "commit_async: %s", + cerr ? rd_kafka_error_string(cerr) + : ""); + async_cnt++; + } + if (acked % 10 == 0) { + cerr = rd_kafka_share_commit_sync( + rkshare, 30000, &partitions); + TEST_ASSERT(!cerr, "commit_sync: %s", + cerr ? rd_kafka_error_string(cerr) + : ""); + RD_IF_FREE( + partitions, + rd_kafka_topic_partition_list_destroy); + partitions = NULL; + sync_cnt++; + } + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + TEST_ASSERT(acked >= target, "expected to ack %d records, got %d", + target, acked); + TEST_SAY( + "acked=%d sync=%d async=%d ack_cb_invocations=%d. " + "Calling close.\n", + acked, sync_cnt, async_cnt, receipts.callback_invocations); + + if (do_close) { + t_start = test_clock(); + close_err = rd_kafka_share_consumer_close(rkshare); + t_close_ms = (test_clock() - t_start) / 1000; + TEST_SAY("close returned in %" PRId64 + " ms (err=%s) " + "ack_cb_invocations=%d\n", + t_close_ms, + close_err ? rd_kafka_error_string(close_err) : "NULL", + receipts.callback_invocations); + if (close_err) + rd_kafka_error_destroy(close_err); + } else { + TEST_SAY( + "Skipping consumer_close (destroy_flags has " + "NO_CONSUMER_CLOSE)\n"); + } + + /* Run destroy on the watchdog thread; fail if it hangs past + * DESTROY_DEADLINE_MS instead of wedging the binary. */ + arg.rkshare = rkshare; + arg.destroy_flags = destroy_flags; + rd_atomic32_init(&arg.done, 0); + + TEST_ASSERT(thrd_create(&destroy_thr, destroy_watchdog_thread, &arg) == + thrd_success, + "Failed to spawn destroy watchdog thread"); + + t_start = test_clock(); + wait_ok = 1; + while (!rd_atomic32_get(&arg.done)) { + if ((test_clock() - t_start) / 1000 >= DESTROY_DEADLINE_MS) { + wait_ok = 0; + break; + } + rd_usleep(100 * 1000, NULL); + } + t_destroy_ms = (test_clock() - t_start) / 1000; + + if (!wait_ok) { + /* Detach so the OS reaps the still-running thread at + * process exit (rkshare permanently leaked). */ + thrd_detach(destroy_thr); + TEST_FAIL( + "rd_kafka_share_destroy hung past %d ms after close " + "(close took %" PRId64 + " ms). acked=%d sync=%d async=%d " + "ack_cb_invocations=%d (expected >= %d).", + DESTROY_DEADLINE_MS, t_close_ms, acked, sync_cnt, async_cnt, + receipts.callback_invocations, async_cnt + sync_cnt); + } + + thrd_join(destroy_thr, NULL); + + TEST_SAY("destroy returned in %" PRId64 " ms (close was %" PRId64 + " ms); final ack_cb_invocations=%d\n", + t_destroy_ms, t_close_ms, receipts.callback_invocations); + + ack_receipts_destroy(&receipts); + + SUB_TEST_PASS(); +} + + +int main_0179_share_consumer_destroy(int argc, char **argv) { + /* Real broker tests */ + test_timeout_set(120); + + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + do_test_destroy_with_subscribe_unsubscribe(0, 0, 0); + do_test_destroy_with_subscribe_unsubscribe( + 1, 0, 0); /* subscribe, no unsubscribe */ + do_test_destroy_with_subscribe_unsubscribe( + 1, 1, 0); /* subscribe then unsubscribe */ + do_test_destroy_with_explicit_ack(0, rd_false /* ack all */); + do_test_destroy_with_explicit_ack(0, rd_true /* ack half */); + do_test_destroy_with_implicit_ack(0); + + do_test_destroy_with_subscribe_unsubscribe( + 1, 0, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + do_test_destroy_with_subscribe_unsubscribe( + 1, 1, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + do_test_destroy_with_subscribe_unsubscribe( + 0, 0, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + do_test_destroy_with_explicit_ack(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, + rd_false); + do_test_destroy_with_explicit_ack(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, + rd_true); + do_test_destroy_with_implicit_ack(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + + do_test_destroy_with_mixed_acks_and_commits(0); + do_test_destroy_with_mixed_acks_and_commits( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} + +int main_0179_share_consumer_destroy_local(int argc, char **argv) { + /* Mock broker tests only (no real broker needed) */ + TEST_SKIP_MOCK_CLUSTER(0); + test_timeout_set(480); + + test_destroy_with_fatal_error(0); + test_destroy_with_cached_acks_and_delayed_broker(0); + test_broker_decommission_with_commit_sync(0, rd_false); + test_broker_decommission_with_commit_sync(0, rd_true); + test_broker_decommission_with_consume_batch(0); + test_broker_decommission_during_close(0, rd_false); + test_broker_decommission_during_close(0, rd_true); + test_broker_decommission_with_commit_async(0, rd_false); + test_broker_decommission_with_commit_async(0, rd_true); + test_destroy_during_rebalance(0); + + test_destroy_with_fatal_error(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + test_destroy_with_cached_acks_and_delayed_broker( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + test_broker_decommission_with_commit_sync( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_false); + test_broker_decommission_with_commit_sync( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_true); + test_broker_decommission_with_consume_batch( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + test_broker_decommission_during_close( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_false); + test_broker_decommission_during_close( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_true); + test_broker_decommission_with_commit_async( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_false); + test_broker_decommission_with_commit_async( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE, rd_true); + test_destroy_during_rebalance(RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + + test_leader_migration_mid_session_destroy(0); + test_leader_migration_mid_session_destroy( + RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0180-share_consumer_config.c b/lib/librdkafka-2.15.0/tests/0180-share_consumer_config.c new file mode 100644 index 00000000000..8ee0f71a83e --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0180-share_consumer_config.c @@ -0,0 +1,1677 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include /* for SIGIO (internal.termination.signal test) */ + +#include "../src/rdkafka_proto.h" /* for RD_KAFKAP_ShareFetch etc. */ + +/** + * @brief Verify that a particular conf shape is rejected by + * rd_kafka_share_consumer_new. + * + * The setter callback is applied to a fresh conf to produce the + * case-under-test (e.g. set rebalance_cb, set events). The helper + * then calls rd_kafka_share_consumer_new and asserts the call returns + * NULL with an errstr containing expected_substr. + */ +static void +verify_share_consumer_conf_set_rejected(const char *case_name, + void (*conf_setter)(rd_kafka_conf_t *), + const char *expected_substr) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + + conf = rd_kafka_conf_new(); + conf_setter(conf); + errstr[0] = '\0'; + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare == NULL, + "[%s] expected NULL share consumer, got non-NULL", + case_name); + TEST_ASSERT(strstr(errstr, expected_substr) != NULL, + "[%s] errstr should mention '%s', got: %s", case_name, + expected_substr, errstr); + TEST_SAY("[%s] rejected with: %s\n", case_name, errstr); + rd_kafka_conf_destroy(conf); +} + +/** + * @brief Verify that setting prop_name to value (via the generic + * rd_kafka_conf_set string interface) causes + * rd_kafka_share_consumer_new to fail with an errstr that + * mentions prop_name. + */ +static void verify_share_consumer_conf_prop_rejected(const char *prop_name, + const char *value) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + rd_kafka_conf_res_t res; + + conf = rd_kafka_conf_new(); + res = rd_kafka_conf_set(conf, prop_name, value, errstr, sizeof(errstr)); + TEST_ASSERT(res == RD_KAFKA_CONF_OK, + "[%s=%s] precondition rd_kafka_conf_set failed: %s", + prop_name, value, errstr); + errstr[0] = '\0'; + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare == NULL, + "[%s=%s] expected NULL share consumer, got non-NULL", + prop_name, value); + TEST_ASSERT(strstr(errstr, prop_name) != NULL, + "[%s=%s] errstr should mention '%s', got: %s", prop_name, + value, prop_name, errstr); + TEST_SAY("[%s=%s] rejected with: %s\n", prop_name, value, errstr); + rd_kafka_conf_destroy(conf); +} + +/** + * @brief Verify that creating a REGULAR (non-share) consumer with + * prop_name=value fails at construction (rd_kafka_new returns + * NULL) with an errstr that mentions prop_name. + * + * group.id is set so the consumer reaches the consumer-specific + * finalize validation. + */ +static void verify_regular_consumer_conf_prop_rejected(const char *prop_name, + const char *value) { + rd_kafka_conf_t *conf; + rd_kafka_t *rk; + char errstr[512]; + + conf = rd_kafka_conf_new(); + TEST_ASSERT(rd_kafka_conf_set(conf, "group.id", "0180-regular", errstr, + sizeof(errstr)) == RD_KAFKA_CONF_OK, + "group.id: %s", errstr); + TEST_ASSERT(rd_kafka_conf_set(conf, prop_name, value, errstr, + sizeof(errstr)) == RD_KAFKA_CONF_OK, + "[%s=%s] precondition rd_kafka_conf_set failed: %s", + prop_name, value, errstr); + + errstr[0] = '\0'; + rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(rk == NULL, + "[%s=%s] expected regular consumer creation to fail, " + "got non-NULL", + prop_name, value); + TEST_ASSERT(strstr(errstr, prop_name) != NULL, + "[%s=%s] errstr should mention '%s', got: %s", prop_name, + value, prop_name, errstr); + TEST_SAY("[%s=%s] regular consumer rejected with: %s\n", prop_name, + value, errstr); + /* rd_kafka_new failed, so it did not take ownership of conf. */ + rd_kafka_conf_destroy(conf); +} + +/* Unused stub used only as a non-NULL function-pointer value for + * rd_kafka_conf_set_rebalance_cb in the rejection test. */ +static void unused_rebalance_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + rd_kafka_topic_partition_list_t *parts, + void *opaque) { +} + +static void setter_rebalance_cb(rd_kafka_conf_t *conf) { + rd_kafka_conf_set_rebalance_cb(conf, unused_rebalance_cb); +} + +static void setter_event_rebalance(rd_kafka_conf_t *conf) { + rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE); +} + +/* Unused stub used only as a non-NULL function-pointer value for + * rd_kafka_conf_set_offset_commit_cb in the rejection test. */ +static void unused_offset_commit_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + rd_kafka_topic_partition_list_t *offsets, + void *opaque) { +} + +static void setter_offset_commit_cb(rd_kafka_conf_t *conf) { + rd_kafka_conf_set_offset_commit_cb(conf, unused_offset_commit_cb); +} + +/* Unused stub used only as a non-NULL function-pointer value for + * rd_kafka_conf_set_consume_cb in the rejection test. */ +static void unused_consume_cb(rd_kafka_message_t *rkmessage, void *opaque) { +} + +static void setter_consume_cb(rd_kafka_conf_t *conf) { + rd_kafka_conf_set_consume_cb(conf, unused_consume_cb); +} + +/* Records the result of attempting to register the various interceptor + * hooks from the on_new interceptor (see interceptor_on_new). All hooks + * are expected to be rejected for share consumers. */ +struct interceptor_add_results { + rd_kafka_resp_err_t on_destroy_err; + rd_kafka_resp_err_t on_consume_err; + rd_kafka_resp_err_t on_commit_err; + rd_kafka_resp_err_t on_send_err; + rd_kafka_resp_err_t on_acknowledgement_err; + rd_kafka_resp_err_t on_request_sent_err; + rd_kafka_resp_err_t on_response_received_err; + rd_kafka_resp_err_t on_thread_start_err; + rd_kafka_resp_err_t on_thread_exit_err; + rd_kafka_resp_err_t on_broker_state_change_err; +}; + +static rd_kafka_resp_err_t unused_on_destroy_ic(rd_kafka_t *rk, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +/* Unused interceptor hooks: only used as non-NULL function pointers in the + * registration attempts. */ +static rd_kafka_resp_err_t unused_on_consume_ic(rd_kafka_t *rk, + rd_kafka_message_t *rkmessage, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t +unused_on_commit_ic(rd_kafka_t *rk, + const rd_kafka_topic_partition_list_t *offsets, + rd_kafka_resp_err_t err, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t unused_on_send_ic(rd_kafka_t *rk, + rd_kafka_message_t *rkmessage, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t +unused_on_acknowledgement_ic(rd_kafka_t *rk, + rd_kafka_message_t *rkmessage, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t unused_on_request_sent_ic(rd_kafka_t *rk, + int sockfd, + const char *brokername, + int32_t brokerid, + int16_t ApiKey, + int16_t ApiVersion, + int32_t CorrId, + size_t size, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t +unused_on_response_received_ic(rd_kafka_t *rk, + int sockfd, + const char *brokername, + int32_t brokerid, + int16_t ApiKey, + int16_t ApiVersion, + int32_t CorrId, + size_t size, + int64_t rtt, + rd_kafka_resp_err_t err, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t +unused_on_thread_start_ic(rd_kafka_t *rk, + rd_kafka_thread_type_t thread_type, + const char *thread_name, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t +unused_on_thread_exit_ic(rd_kafka_t *rk, + rd_kafka_thread_type_t thread_type, + const char *thread_name, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +static rd_kafka_resp_err_t +unused_on_broker_state_change_ic(rd_kafka_t *rk, + int32_t broker_id, + const char *secproto, + const char *host, + int port, + const char *state, + void *ic_opaque) { + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +/* on_new interceptor: tries to register one interceptor of each hook + * category (the only point at which the rk-level hooks may be added) and + * records the per-hook return codes. For share consumers every hook is + * expected to be rejected. Returns NO_ERROR itself so construction is not + * failed by this interceptor. */ +static rd_kafka_resp_err_t interceptor_on_new(rd_kafka_t *rk, + const rd_kafka_conf_t *conf, + void *ic_opaque, + char *errstr, + size_t errstr_size) { + struct interceptor_add_results *r = ic_opaque; + + r->on_destroy_err = rd_kafka_interceptor_add_on_destroy( + rk, "test-on-destroy", unused_on_destroy_ic, NULL); + r->on_consume_err = rd_kafka_interceptor_add_on_consume( + rk, "test-on-consume", unused_on_consume_ic, NULL); + r->on_commit_err = rd_kafka_interceptor_add_on_commit( + rk, "test-on-commit", unused_on_commit_ic, NULL); + r->on_send_err = rd_kafka_interceptor_add_on_send( + rk, "test-on-send", unused_on_send_ic, NULL); + r->on_acknowledgement_err = rd_kafka_interceptor_add_on_acknowledgement( + rk, "test-on-acknowledgement", unused_on_acknowledgement_ic, NULL); + r->on_request_sent_err = rd_kafka_interceptor_add_on_request_sent( + rk, "test-on-request-sent", unused_on_request_sent_ic, NULL); + r->on_response_received_err = + rd_kafka_interceptor_add_on_response_received( + rk, "test-on-response-received", unused_on_response_received_ic, + NULL); + r->on_thread_start_err = rd_kafka_interceptor_add_on_thread_start( + rk, "test-on-thread-start", unused_on_thread_start_ic, NULL); + r->on_thread_exit_err = rd_kafka_interceptor_add_on_thread_exit( + rk, "test-on-thread-exit", unused_on_thread_exit_ic, NULL); + r->on_broker_state_change_err = + rd_kafka_interceptor_add_on_broker_state_change( + rk, "test-on-broker-state-change", + unused_on_broker_state_change_ic, NULL); + + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +/** + * @brief Share consumer has no rebalance callback semantics; the + * factory rejects rebalance_cb at construction so an app's + * handler can never silently never-fire. + */ +static void test_rebalance_cb_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_set_rejected( + "rebalance_cb set", setter_rebalance_cb, "rebalance_cb"); + + SUB_TEST_PASS(); +} + +/** + * @brief Same reasoning as rebalance_cb — the event-mask form of + * opting into rebalance delivery is also rejected. + */ +static void test_event_rebalance_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_set_rejected("RD_KAFKA_EVENT_REBALANCE set", + setter_event_rebalance, + "RD_KAFKA_EVENT_REBALANCE"); + + SUB_TEST_PASS(); +} + +/** + * @brief `offset_commit_cb` reports completion of offset commits. Share + * consumers do not commit offsets (they acknowledge records, with + * enable.auto.commit forced off), so the callback never fires on + * the share path; it is rejected at construction. The Java share + * consumer likewise has no OffsetCommitCallback (its analog is + * AcknowledgementCommitCallback). + */ +static void test_offset_commit_cb_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_set_rejected("offset_commit_cb set", + setter_offset_commit_cb, + "offset_commit_cb"); + + SUB_TEST_PASS(); +} + +/** + * @brief `consume_cb` delivers messages via the classic + * rd_kafka_consumer_poll()/rd_kafka_consume_callback*() dispatch + * path. Share consumers use the batch-returning rd_kafka_share_poll() + * and never invoke this callback, so it is rejected at construction. + * Java has no per-message consume callback for either consumer type. + */ +static void test_consume_cb_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_set_rejected( + "consume_cb set", setter_consume_cb, "consume_cb"); + + SUB_TEST_PASS(); +} + +/** + * @brief Interceptors are not supported for share consumers: registering + * any handle-level interceptor hook (the only opportunity is from an + * on_new interceptor) returns RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED. + * This mirrors the Java share consumer rejecting ConsumerInterceptor. + * The plugin loader (and the on_new interceptor itself) still run, so + * construction succeeds; only the hook registrations are rejected. + */ +static void test_interceptors_rejected_for_share_consumer(void) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + struct interceptor_add_results results; + char errstr[512]; + + SUB_TEST_QUICK(); + + memset(&results, 0, sizeof(results)); + + conf = rd_kafka_conf_new(); + rd_kafka_conf_interceptor_add_on_new(conf, "test-on-new", + interceptor_on_new, &results); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, + "share consumer construction should still succeed: %s", + errstr); + +#define ASSERT_IC_REJECTED(field) \ + TEST_ASSERT(results.field == RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, \ + "expected " #field \ + " to be rejected with " \ + "_NOT_IMPLEMENTED, got %s", \ + rd_kafka_err2name(results.field)) + + ASSERT_IC_REJECTED(on_destroy_err); + ASSERT_IC_REJECTED(on_consume_err); + ASSERT_IC_REJECTED(on_commit_err); + ASSERT_IC_REJECTED(on_send_err); + ASSERT_IC_REJECTED(on_acknowledgement_err); + ASSERT_IC_REJECTED(on_request_sent_err); + ASSERT_IC_REJECTED(on_response_received_err); + ASSERT_IC_REJECTED(on_thread_start_err); + ASSERT_IC_REJECTED(on_thread_exit_err); + ASSERT_IC_REJECTED(on_broker_state_change_err); + +#undef ASSERT_IC_REJECTED + + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +} + +/** + * @brief `consume.callback.max.messages` caps the batch size dispatched + * per rd_kafka_consume_callback*() call. That callback path is not + * used by share consumers (which use rd_kafka_share_poll()), so the + * property is rejected at construction. No Java equivalent. + */ +static void test_consume_callback_max_messages_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected( + "consume.callback.max.messages", "100"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer locks enable.auto.commit to false internally. + * Any explicit set by the app (true or false) is rejected so + * the value can only come from the library default. + */ +static void test_enable_auto_commit_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("enable.auto.commit", "true"); + verify_share_consumer_conf_prop_rejected("enable.auto.commit", "false"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer requires group.protocol=consumer (forced + * internally); any explicit set by the app — even to the + * same value — is rejected so the value can only come from + * the library. + */ +static void test_group_protocol_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("group.protocol", "consumer"); + verify_share_consumer_conf_prop_rejected("group.protocol", "classic"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer forces socket.max.fails=1 to keep the broker + * share session in sync; any explicit set by the app is + * rejected. + */ +static void test_socket_max_fails_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("socket.max.fails", "1"); + verify_share_consumer_conf_prop_rejected("socket.max.fails", "5"); + + SUB_TEST_PASS(); +} + +/** + * @brief For non-share (regular) consumers, fetch.min.bytes must stay in + * the historical range 1..100000000. The property bounds were + * widened to 0..INT_MAX for share-consumer parity with the Java + * client; regular consumers are re-restricted at finalize, so 0 + * and values above 100000000 (e.g. INT_MAX) are rejected. + */ +static void test_fetch_min_bytes_regular_consumer_range_rejected(void) { + SUB_TEST_QUICK(); + + /* 0 is below the regular-consumer minimum of 1. */ + verify_regular_consumer_conf_prop_rejected("fetch.min.bytes", "0"); + /* INT_MAX exceeds the regular-consumer maximum of 100000000. */ + verify_regular_consumer_conf_prop_rejected("fetch.min.bytes", + "2147483647"); + verify_regular_consumer_conf_prop_rejected("fetch.min.bytes", + "100000001"); + + SUB_TEST_PASS(); +} + +/** + * @brief Offset-reset for share consumer is a broker-side share- + * group property (`share.auto.offset.reset`); the client + * `auto.offset.reset` is not used. Any explicit set on the + * client is rejected. + */ +static void test_auto_offset_reset_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("auto.offset.reset", + "earliest"); + verify_share_consumer_conf_prop_rejected("auto.offset.reset", "latest"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer forces group.protocol=consumer, so the + * downstream consumer-protocol validation rejects + * session.timeout.ms (defined broker side). Verify the + * rejection surfaces for share consumer too. + */ +static void test_session_timeout_ms_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("session.timeout.ms", "30000"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer forces group.protocol=consumer, so the + * downstream consumer-protocol validation rejects + * partition.assignment.strategy. Verify the rejection + * surfaces for share consumer too. + */ +static void test_partition_assignment_strategy_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected( + "partition.assignment.strategy", "range"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer forces group.protocol=consumer, so the + * downstream consumer-protocol validation rejects + * group.protocol.type. Verify the rejection surfaces for + * share consumer too. + */ +static void test_group_protocol_type_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("group.protocol.type", + "consumer"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share consumer forces group.protocol=consumer, so the + * downstream consumer-protocol validation rejects + * heartbeat.interval.ms (defined broker side). Verify the + * rejection surfaces for share consumer too. + */ +static void test_heartbeat_interval_ms_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("heartbeat.interval.ms", + "3000"); + + SUB_TEST_PASS(); +} + +/** + * @brief For receive.message.max.bytes, an explicit value below + * `fetch.max.bytes` + 512 is rejected at construction so the receive cap can + * always hold a full fetch response. Here `fetch.max.bytes` is left at its + * default + * (~50 MB), so any value below that + 512 (e.g. 1 MB) is rejected. + */ +static void +test_receive_message_max_bytes_below_floor_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("receive.message.max.bytes", + "1000000"); + + SUB_TEST_PASS(); +} + +/** + * @brief `reconnect.backoff.max.ms` must be >= `reconnect.backoff.ms`; + * a smaller max is rejected at construction. This is a common + * (non-share-specific) finalize check, exercised here for the + * share consumer factory. + */ +static void +test_reconnect_backoff_max_below_min_rejected_at_construction(void) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + + SUB_TEST_QUICK(); + + conf = rd_kafka_conf_new(); + TEST_ASSERT(rd_kafka_conf_set(conf, "reconnect.backoff.ms", "1000", + errstr, + sizeof(errstr)) == RD_KAFKA_CONF_OK, + "reconnect.backoff.ms: %s", errstr); + TEST_ASSERT(rd_kafka_conf_set(conf, "reconnect.backoff.max.ms", "500", + errstr, + sizeof(errstr)) == RD_KAFKA_CONF_OK, + "reconnect.backoff.max.ms: %s", errstr); + + errstr[0] = '\0'; + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare == NULL, + "expected NULL share consumer when " + "reconnect.backoff.max.ms < reconnect.backoff.ms"); + TEST_ASSERT(strstr(errstr, "reconnect.backoff.max.ms") != NULL, + "errstr should mention reconnect.backoff.max.ms, got: %s", + errstr); + TEST_SAY("rejected with: %s\n", errstr); + rd_kafka_conf_destroy(conf); + + SUB_TEST_PASS(); +} + +/** + * @brief Static membership (`group.instance.id`) is not supported for + * share groups; the property is rejected for share consumers. + */ +static void test_group_instance_id_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("group.instance.id", + "share-instance-1"); + + SUB_TEST_PASS(); +} + +/** + * @brief `isolation.level` is defined broker-side for share groups; the + * client property is rejected for share consumers. + */ +static void test_isolation_level_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("isolation.level", + "read_committed"); + + SUB_TEST_PASS(); +} + +/** + * @brief Share-group assignment is broker-driven; the client + * `group.remote.assignor` property is rejected for share + * consumers. + */ +static void test_group_remote_assignor_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("group.remote.assignor", + "uniform"); + + SUB_TEST_PASS(); +} + +/** + * @brief `queued.min.messages` tunes the per-partition prefetch queue, + * which share consumers (broker-driven via max.poll.records) do + * not use; it is rejected for share consumers. + */ +static void test_queued_min_messages_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("queued.min.messages", "1000"); + + SUB_TEST_PASS(); +} + +/** + * @brief `queued.max.messages.kbytes` tunes the per-partition prefetch + * queue, which share consumers do not use; it is rejected for + * share consumers. + */ +static void test_queued_max_messages_kbytes_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("queued.max.messages.kbytes", + "1024"); + + SUB_TEST_PASS(); +} + +/** + * @brief `fetch.queue.backoff.ms` backs off the per-partition prefetch + * queue throttle (queued.min.messages / queued.max.messages.kbytes), + * which share consumers do not use; it is rejected for share + * consumers. No Java equivalent. + */ +static void test_fetch_queue_backoff_ms_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("fetch.queue.backoff.ms", + "1000"); + + SUB_TEST_PASS(); +} + +/** + * @brief `fetch.error.backoff.ms` postpones the next Fetch after a fetch + * error, but it is only consulted on the regular (non-share) Fetch + * RPC path (rd_kafka_broker_fetch_backoff / + * rd_kafka_toppar_fetch_backoff). The ShareFetch path does not use + * it: ShareFetch/ShareAcknowledge RPC-level errors are not retried + * with a fetch backoff (they are handled via session reset or the + * next poll), and connection failures reconnect via + * reconnect.backoff.ms. It is also librdkafka-specific (no Java + * equivalent; Java uses the generic retry.backoff.ms). So it is + * rejected for share consumers. + */ +static void test_fetch_error_backoff_ms_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("fetch.error.backoff.ms", + "1000"); + + SUB_TEST_PASS(); +} + +/** + * @brief `enable.partition.eof` emits a synthetic PARTITION_EOF event + * when the client's per-partition fetch cursor reaches the + * broker-reported high-water mark. Share consumers do not + * maintain a per-partition fetch cursor (the broker manages the + * share-group offset / acquired ranges), so the EOF event is + * never produced on the ShareFetch path. librdkafka-specific (no + * Java equivalent). It is rejected for share consumers. + */ +static void test_enable_partition_eof_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("enable.partition.eof", + "true"); + + SUB_TEST_PASS(); +} + +/** + * @brief `message.copy.max.bytes` is a producer-only copy/zero-copy + * threshold used solely when building ProduceRequests. It is + * flagged _RK_GLOBAL (not _RK_PRODUCER), so unlike linger.ms / + * batch.* it is silently accepted by consumers rather than warned + * about; it does nothing on the share-consumer path. Rejected for + * share consumers. No Java equivalent. + */ +static void test_message_copy_max_bytes_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("message.copy.max.bytes", + "1000"); + + SUB_TEST_PASS(); +} + +/** + * @brief `topic.blacklist` is a librdkafka-specific metadata filter (no + * Java equivalent) that drops matching topics from broker metadata + * "as if they did not exist". The Apache Kafka Java share consumer + * has no such config, so to keep the share-consumer config surface + * on par with the Java client it is rejected for share consumers. + */ +static void test_topic_blacklist_rejected_at_construction(void) { + SUB_TEST_QUICK(); + + verify_share_consumer_conf_prop_rejected("topic.blacklist", "^test.*"); + + SUB_TEST_PASS(); +} + +struct idle_reconnect_counters { + rd_atomic32_t idle_closes; + rd_atomic32_t reconnects_after_idle; +}; + +/* =================================================================== + * Log callback for the fetch-connection idle test. Same as + * idle_reconnect_log_cb but filtered to the FETCH (leader) connection + * only: the broker name is prefixed into buf, so coordinator lines + * contain "GroupCoordinator" and fetch-broker lines do not. + * =================================================================== */ +static void fetch_idle_log_cb(const rd_kafka_t *rk, + int level, + const char *fac, + const char *buf) { + struct idle_reconnect_counters *c = rd_kafka_opaque(rk); + if (!c || strstr(buf, "GroupCoordinator")) + return; /* ignore the coordinator connection */ + if (!strcmp(fac, "FAIL") && + strstr(buf, "Connection max idle time exceeded")) + rd_atomic32_add(&c->idle_closes, 1); + else if (!strcmp(fac, "CONNECT") && strstr(buf, "connecting") && + rd_atomic32_get(&c->idle_closes) > 0) + rd_atomic32_add(&c->reconnects_after_idle, 1); +} + +/** + * @brief Build a producer with linger.ms=0 so each rd_kafka_produce() + * flushes as its own broker batch. + */ +static rd_kafka_t *create_no_linger_producer(void) { + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + TEST_ASSERT(rd_kafka_conf_set(conf, "linger.ms", "0", errstr, + sizeof(errstr)) == RD_KAFKA_CONF_OK, + "linger.ms=0: %s", errstr); + return test_create_handle(RD_KAFKA_PRODUCER, conf); +} + + +/** + * @brief Build a share consumer for \p group_id, optionally setting one + * extra config property \p prop to \p value. + */ +static rd_kafka_share_t *create_share_consumer_with_prop(const char *group_id, + const char *prop, + int value) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + char val[32]; + + test_conf_init(&conf, NULL, 0); + rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr)); + + if (prop) { + rd_snprintf(val, sizeof(val), "%d", value); + TEST_ASSERT(rd_kafka_conf_set(conf, prop, val, errstr, + sizeof(errstr)) == + RD_KAFKA_CONF_OK, + "%s=%d: %s", prop, value, errstr); + } + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + return rkshare; +} + + +/** + * @brief Produce \p msgcnt records to \p topic, partition 0, with + * \p gap_ms between each produce call. With linger.ms=0 on the + * producer this lands each record as its own broker batch. + */ +static void produce_one_per_batch(rd_kafka_t *producer, + const char *topic, + int msgcnt, + int gap_ms) { + rd_kafka_topic_t *rkt; + rd_kafka_resp_err_t err; + char payload[64]; + int i; + + rkt = rd_kafka_topic_new(producer, topic, NULL); + TEST_ASSERT(rkt, "topic_new(%s) failed: %s", topic, + rd_kafka_err2str(rd_kafka_last_error())); + + for (i = 0; i < msgcnt; i++) { + rd_snprintf(payload, sizeof(payload), "msg-%d", i); + if (rd_kafka_produce(rkt, 0, RD_KAFKA_MSG_F_COPY, payload, + strlen(payload), NULL, 0, NULL) == -1) + TEST_FAIL("produce #%d failed: %s", i, + rd_kafka_err2str(rd_kafka_last_error())); + + err = rd_kafka_flush(producer, 30 * 1000); + TEST_ASSERT(!err, "flush after produce #%d: %s", i, + rd_kafka_err2str(err)); + + if (i + 1 < msgcnt) + rd_usleep(gap_ms * 1000, NULL); + } + + rd_kafka_topic_destroy(rkt); +} + + +/** + * @brief Consume \p target records and record how many records came in + * each batch returned by rd_kafka_share_poll(). + * + * @returns Number of batches it took to reach \p target records. + * Out-param \p batch_sizes is filled with each batch's + * record count, up to \p batch_sizes_cap entries. + */ +static int consume_record_batches(rd_kafka_share_t *rkshare, + int target, + int *batch_sizes, + int batch_sizes_cap, + int per_call_timeout_ms, + int max_calls) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + size_t j; + rd_kafka_error_t *error; + int batches = 0; + int got = 0; + int call; + + for (call = 0; call < max_calls && got < target; call++) { + error = + rd_kafka_share_poll(rkshare, per_call_timeout_ms, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + if (rcvd == 0) { + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + + if (batches < batch_sizes_cap) + batch_sizes[batches] = (int)rcvd; + batches++; + + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + got++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + return batches; +} + +/** + * @brief Drain \p msgcnt records from \p topic with a share consumer + * configured by \p set_fetch_max_bytes / \p fetch_max_bytes, and + * return how many non-empty poll batches it took. + * + * max.poll.records is left at the library default so that the only + * thing capping a single poll batch is the byte limit under test. + */ +static int drain_count_with_fetch_max_bytes(const char *group, + const char *topic, + int msgcnt, + rd_bool_t set_fetch_max_bytes, + int fetch_max_bytes) { + rd_kafka_share_t *rkshare; + int batch_sizes[256] = {0}; + int batches; + + rkshare = create_share_consumer_with_prop( + group, set_fetch_max_bytes ? "fetch.max.bytes" : NULL, + fetch_max_bytes); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + batches = + consume_record_batches(rkshare, msgcnt, batch_sizes, + (int)RD_ARRAY_SIZE(batch_sizes), 3000, 200); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + return batches; +} + +/** + * @brief Produce a single record of \p value_size bytes to \p topic + * partition 0. The default producer message.max.bytes (1 MB) and + * broker max.message.bytes (~1 MB) both comfortably exceed the + * sizes used here, so no size overrides are needed. + */ +static void +produce_one_sized(rd_kafka_t *producer, const char *topic, int value_size) { + rd_kafka_topic_t *rkt; + char *payload; + + rkt = rd_kafka_topic_new(producer, topic, NULL); + TEST_ASSERT(rkt, "topic_new(%s) failed: %s", topic, + rd_kafka_err2str(rd_kafka_last_error())); + + payload = rd_malloc(value_size); + memset(payload, 'x', value_size); + + if (rd_kafka_produce(rkt, 0, RD_KAFKA_MSG_F_COPY, payload, value_size, + NULL, 0, NULL) == -1) + TEST_FAIL("produce (%d bytes) failed: %s", value_size, + rd_kafka_err2str(rd_kafka_last_error())); + TEST_ASSERT(!rd_kafka_flush(producer, 30 * 1000), "flush timed out"); + + rd_free(payload); + rd_kafka_topic_destroy(rkt); +} + +/* Set when the receive-cap test sees the expected _BAD_MSG error via the + * client error_cb. The receive-side size check fails the broker + * connection (rdkafka_broker.c), so the error is delivered to error_cb, + * not as the rd_kafka_share_poll() return value. */ +static rd_atomic32_t recv_max_bad_msg_seen; + +/* is_fatal_cb for the receive-cap test: the oversized-response + * _BAD_MSG is expected (not test-fatal); record it and let the test + * continue. Any other error is still treated as fatal. */ +static int recv_max_is_fatal_cb(rd_kafka_t *rk, + rd_kafka_resp_err_t err, + const char *reason) { + if (err == RD_KAFKA_RESP_ERR__BAD_MSG && reason && + strstr(reason, "Invalid response size")) { + rd_atomic32_add(&recv_max_bad_msg_seen, 1); + return 0; /* not fatal */ + } + if (err == RD_KAFKA_RESP_ERR__TRANSPORT || + err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) + return 0; /* expected cascade from the receive-cap failure */ + return 1; /* fatal */ +} + +/** + * @brief Verify max.poll.records=5 splits 10 single-record broker + * batches into 2 consume_batch() returns of ~5 records each. + * + * Setup: producer with linger.ms=0 emits 10 records with 500ms gap, so + * each lands as its own broker batch. Consumer with + * max.poll.records=5: across all consume_batch() calls, no single call + * returns more than 5 records and the 10 records are drained in 2 or + * more batches (the cap is the upper bound the lib must respect). + */ +static void test_max_poll_records_caps_batch_at_5(void) { + const char *topic; + const char *group = "0180-max-poll-records-5"; + rd_kafka_t *producer; + rd_kafka_share_t *rkshare; + const int msgcnt = 10; + const int max_poll = 5; + int batch_sizes[32] = {0}; + int batches; + int i; + + SUB_TEST(); + + producer = create_no_linger_producer(); + + topic = test_mk_topic_name("0180-max-poll-5", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + produce_one_per_batch(producer, topic, msgcnt, 500); + + rkshare = create_share_consumer_with_prop(group, "max.poll.records", + max_poll); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + batches = + consume_record_batches(rkshare, msgcnt, batch_sizes, + (int)RD_ARRAY_SIZE(batch_sizes), 3000, 30); + + TEST_SAY("max.poll.records=%d, msgcnt=%d -> %d batch(es):", max_poll, + msgcnt, batches); + for (i = 0; i < batches && i < (int)RD_ARRAY_SIZE(batch_sizes); i++) + TEST_SAY0(" %d", batch_sizes[i]); + TEST_SAY0("\n"); + + for (i = 0; i < batches && i < (int)RD_ARRAY_SIZE(batch_sizes); i++) + TEST_ASSERT(batch_sizes[i] <= max_poll, + "Batch #%d returned %d records, exceeds " + "max.poll.records=%d", + i, batch_sizes[i], max_poll); + + TEST_ASSERT(batches >= 2, + "Expected at least 2 batches with max.poll.records=%d " + "for %d records, got %d", + max_poll, msgcnt, batches); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Verify max.poll.records=10 drains 10 single-record broker + * batches in a single consume_batch() call. + */ +static void test_max_poll_records_allows_full_drain_at_10(void) { + const char *topic; + const char *group = "0180-max-poll-records-10"; + rd_kafka_t *producer; + rd_kafka_share_t *rkshare; + const int msgcnt = 10; + const int max_poll = 10; + int batch_sizes[32] = {0}; + int batches; + int i; + + SUB_TEST(); + + producer = create_no_linger_producer(); + + topic = test_mk_topic_name("0180-max-poll-10", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + produce_one_per_batch(producer, topic, msgcnt, 500); + + rkshare = create_share_consumer_with_prop(group, "max.poll.records", + max_poll); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + batches = + consume_record_batches(rkshare, msgcnt, batch_sizes, + (int)RD_ARRAY_SIZE(batch_sizes), 5000, 30); + + TEST_SAY("max.poll.records=%d, msgcnt=%d -> %d batch(es):", max_poll, + msgcnt, batches); + for (i = 0; i < batches && i < (int)RD_ARRAY_SIZE(batch_sizes); i++) + TEST_SAY0(" %d", batch_sizes[i]); + TEST_SAY0("\n"); + + { + int sum = 0; + for (i = 0; i < batches && i < (int)RD_ARRAY_SIZE(batch_sizes); + i++) { + TEST_ASSERT(batch_sizes[i] <= max_poll, + "Batch #%d returned %d records, " + "exceeds max.poll.records=%d", + i, batch_sizes[i], max_poll); + sum += batch_sizes[i]; + } + TEST_ASSERT(sum == msgcnt, + "Expected %d total records across batches, " + "got %d", + msgcnt, sum); + } + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + +/** + * @brief Verify fetch.max.bytes behaves as a soft per-fetch byte limit + * for share consumers. + * + * Produces msgcnt single-record broker batches (linger.ms=0 producer + * with a gap between produces). Then: + * + * - With fetch.max.bytes UNSET (library default 50 MB): the broker can + * pack many batches into one ShareFetch response, so the records + * drain in far fewer poll calls than there are batches. + * + * - With fetch.max.bytes=1: the limit is a soft cap and the broker's + * "return at least one batch" guarantee yields + * exactly one broker batch per ShareFetch response, so draining + * takes one poll per produced batch. + * The two phases use distinct topics/groups so their share sessions and + * offsets do not interfere. + */ +static void test_fetch_max_bytes_one_is_soft_limit(void) { + const char *topic_default; + const char *topic_one; + const int msgcnt = 10; + rd_kafka_t *producer; + int drains_default; + int drains_one; + + SUB_TEST(); + + producer = create_no_linger_producer(); + + /* Phase 1: default fetch.max.bytes. */ + topic_default = test_mk_topic_name("0180-fmb-default", 1); + test_create_topic_wait_exists(NULL, topic_default, 1, -1, 60 * 1000); + produce_one_per_batch(producer, topic_default, msgcnt, 200); + + drains_default = drain_count_with_fetch_max_bytes( + "0180-fetch-max-bytes-default", topic_default, msgcnt, + rd_false /*unset*/, 0); + TEST_SAY( + "fetch.max.bytes default: %d record(s) drained in %d poll " + "batch(es)\n", + msgcnt, drains_default); + + /* Phase 2: fetch.max.bytes=1. */ + topic_one = test_mk_topic_name("0180-fmb-one", 1); + test_create_topic_wait_exists(NULL, topic_one, 1, -1, 60 * 1000); + produce_one_per_batch(producer, topic_one, msgcnt, 200); + + drains_one = drain_count_with_fetch_max_bytes( + "0180-fetch-max-bytes-one", topic_one, msgcnt, rd_true /*set*/, 1); + TEST_SAY( + "fetch.max.bytes=1: %d record(s) drained in %d poll " + "batch(es)\n", + msgcnt, drains_one); + + /* With fetch.max.bytes=1 the broker returns one batch per fetch, + * so the number of poll batches equals the number of produced + * broker batches (msgcnt single-record batches). */ + TEST_ASSERT(drains_one == msgcnt, + "fetch.max.bytes=1 should drain one batch per fetch: " + "expected %d poll batches, got %d", + msgcnt, drains_one); + + /* The default (large) limit must drain in strictly fewer poll + * batches than the one-per-fetch case. */ + TEST_ASSERT(drains_default < drains_one, + "default fetch.max.bytes (%d poll batches) should drain " + "in fewer batches than fetch.max.bytes=1 (%d poll " + "batches)", + drains_default, drains_one); + + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + + +/** + * @brief Drain msgcnt records from a fresh topic with a share consumer + * configured with fetch.min.bytes=\p value, and assert all + * records are received. + * + * Confirms fetch.min.bytes is functional end-to-end for share consumers + * across its range: + * - 0 : broker responds immediately (no long-poll). + * - INT_MAX : broker can never satisfy the threshold, so it responds + * when fetch.max.wait.ms elapses. + * In both cases all records must still drain (matches the Java client). + */ +static void verify_share_fetch_min_bytes_drains_all(const char *group, + const char *topic_suffix, + int value) { + const char *topic; + const int msgcnt = 10; + rd_kafka_t *producer; + rd_kafka_share_t *rkshare; + int batch_sizes[64] = {0}; + int batches; + int i; + int total = 0; + + producer = create_no_linger_producer(); + + topic = test_mk_topic_name(topic_suffix, 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + produce_one_per_batch(producer, topic, msgcnt, 200); + + rkshare = + create_share_consumer_with_prop(group, "fetch.min.bytes", value); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + batches = + consume_record_batches(rkshare, msgcnt, batch_sizes, + (int)RD_ARRAY_SIZE(batch_sizes), 3000, 60); + + for (i = 0; i < batches && i < (int)RD_ARRAY_SIZE(batch_sizes); i++) + total += batch_sizes[i]; + + TEST_SAY( + "fetch.min.bytes=%d: drained %d/%d record(s) in %d poll " + "batch(es)\n", + value, total, msgcnt, batches); + + TEST_ASSERT(total == msgcnt, + "fetch.min.bytes=%d should drain all records: expected " + "%d, got %d", + value, msgcnt, total); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_destroy(producer); +} + +/** + * @brief fetch.min.bytes=0 is functional for share consumers (broker + * responds immediately, no long-poll). Matches the Java client. + */ +static void test_fetch_min_bytes_zero_drains_all(void) { + SUB_TEST(); + verify_share_fetch_min_bytes_drains_all("0180-fetch-min-bytes-zero", + "0180-fmin-zero", 0); + SUB_TEST_PASS(); +} + +/** + * @brief fetch.min.bytes=INT_MAX is functional for share consumers: the + * broker can't satisfy the threshold, so it responds when + * fetch.max.wait.ms elapses and all records still drain. Confirms + * the INT_MAX upper bound (raised for Java parity) works + * end-to-end. + */ +static void test_fetch_min_bytes_max_drains_all(void) { + SUB_TEST(); + verify_share_fetch_min_bytes_drains_all("0180-fetch-min-bytes-max", + "0180-fmin-max", 2147483647); + SUB_TEST_PASS(); +} + + +/** + * @brief Verify fetch.max.bytes=0 returns no records for a share consumer. + * + * fetch.max.bytes=0 is accepted, but a ShareFetch with MaxBytes=0 returns + * no records. This matches the Java client. + * This test pins the resulting behavior so it does not silently change. + */ +static void test_fetch_max_bytes_zero_returns_no_records(void) { + const char *topic; + const char *group = "0180-fetch-max-bytes-zero"; + const int msgcnt = 10; + rd_kafka_t *producer; + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + int received = 0; + int call; + + SUB_TEST(); + + producer = create_no_linger_producer(); + + topic = test_mk_topic_name("0180-fmax-zero", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + produce_one_per_batch(producer, topic, msgcnt, 200); + + rkshare = create_share_consumer_with_prop(group, "fetch.max.bytes", 0); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Poll a bounded number of times; with fetch.max.bytes=0 the broker + * returns no records, so every poll must be empty. */ + for (call = 0; call < 5; call++) { + rd_kafka_error_t *error = + rd_kafka_share_poll(rkshare, 1000, &batch); + TEST_ASSERT(!error, "fetch.max.bytes=0 poll #%d failed: %s", + call, error ? rd_kafka_error_string(error) : ""); + received += (int)rd_kafka_messages_count(batch); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_SAY("fetch.max.bytes=0: received %d record(s) over 5 polls\n", + received); + + TEST_ASSERT(received == 0, + "fetch.max.bytes=0 should return no records, got %d", + received); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_destroy(producer); + + SUB_TEST_PASS(); +} + +/** + * @brief Characterize the FETCH connection's idle behavior after the + * consumer has drained all records and stopped polling. + * + * While records are available the share fetch loop self-perpetuates and + * the fetch connection stays busy. Once all records are consumed, + * share_fetch_more_records clears and no further ShareFetch is enqueued, + * so the fetch (leader) connection goes idle. This test produces a few + * records, drains them, then stops polling and sleeps past + * connections.max.idle.ms, and asserts the fetch connection idle-closed + * and then reconnected (current behavior: a persistent connection is + * requested while the broker has an assignment and is not UP). + */ +static void test_share_consumer_fetch_conn_idle_after_drain(void) { + const char *topic; + const char *group = "0180-fetch-idle"; + const int msgcnt = 5; + const int idle_ms = 5000; + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + rd_kafka_t *producer; + struct idle_reconnect_counters counters; + rd_kafka_messages_t *batch = NULL; + char errstr[512]; + int drained = 0; + int i; + + SUB_TEST(); + + rd_atomic32_init(&counters.idle_closes, 0); + rd_atomic32_init(&counters.reconnects_after_idle, 0); + + producer = create_no_linger_producer(); + topic = test_mk_topic_name("0180-fetch-idle", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + produce_one_per_batch(producer, topic, msgcnt, 100); + + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "debug", "all"); + /* Short idle timeout so the test runs fast (overrides the 9-min + * share-consumer default). */ + test_conf_set(conf, "connections.max.idle.ms", "5000"); + rd_kafka_conf_set_log_cb(conf, fetch_idle_log_cb); + rd_kafka_conf_set_opaque(conf, &counters); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Drain all records so share_fetch_more_records clears and the + * fetch loop quiesces. */ + for (i = 0; i < 30 && drained < msgcnt; i++) { + rd_kafka_error_t *error = + rd_kafka_share_poll(rkshare, 1000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + drained += (int)rd_kafka_messages_count(batch); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(drained >= msgcnt, "expected to drain %d records, got %d", + msgcnt, drained); + + /* Stop polling and idle past the timeout (~2x) so the fetch + * connection can be closed by connections.max.idle.ms. */ + rd_sleep((idle_ms * 2) / 1000 + 2); + + TEST_SAY("fetch-conn idle closes=%d, reconnects-after-idle=%d\n", + rd_atomic32_get(&counters.idle_closes), + rd_atomic32_get(&counters.reconnects_after_idle)); + + TEST_ASSERT(rd_atomic32_get(&counters.idle_closes) >= 1, + "expected the idle fetch connection to be closed by " + "connections.max.idle.ms, got %d closes", + rd_atomic32_get(&counters.idle_closes)); + TEST_ASSERT(rd_atomic32_get(&counters.reconnects_after_idle) >= 1, + "expected the fetch connection to reconnect after the " + "idle close (persistent connection requested while the " + "broker has an assignment), got %d", + rd_atomic32_get(&counters.reconnects_after_idle)); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_destroy(producer); +} + +/** + * @brief A user-set `receive.message.max.bytes` is honored at runtime: + * a record whose batch fits under the cap is consumed normally, + * but a record whose batch alone exceeds the cap trips the + * receive-side size check + * (`Invalid response size ... increase receive.message.max.bytes`, + * rdkafka_broker.c). That fails the broker connection, so the + * error is delivered as RD_KAFKA_RESP_ERR__BAD_MSG via the client + * error_cb (captured here by recv_max_is_fatal_cb), rather than as + * the rd_kafka_share_poll() return. + * + * `fetch.max.bytes` is set below `receive.message.max.bytes` - 512 so the + * config is accepted; the broker still returns at least one full batch per + * fetch (minOneMessage), so an oversized batch is delivered in full and + * overflows the receive cap. + */ +static void test_receive_message_max_bytes_is_honored(void) { + const char *topic; + const char *group = "0180-recv-max-bytes"; + const int recv_max = 100000; /* 100 KB receive cap */ + const int small_sz = 1000; /* fits under the cap */ + const int big_sz = 200000; /* one batch exceeds the cap */ + rd_kafka_conf_t *conf; + rd_kafka_t *producer; + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + char errstr[512]; + char val[32]; + int small_consumed = 0; + int call; + + SUB_TEST(); + + rd_atomic32_init(&recv_max_bad_msg_seen, 0); + /* The oversized-response _BAD_MSG arrives via error_cb and would + * otherwise fail the test; mark it as expected. */ + test_curr->is_fatal_cb = recv_max_is_fatal_cb; + + topic = test_mk_topic_name("0180-recv-max-bytes", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + + /* Small record first, then the oversized one. */ + producer = create_no_linger_producer(); + produce_one_sized(producer, topic, small_sz); + produce_one_sized(producer, topic, big_sz); + rd_kafka_destroy(producer); + + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "group.id", group); + rd_snprintf(val, sizeof(val), "%d", recv_max); + test_conf_set(conf, "receive.message.max.bytes", val); + /* Must be <= receive.message.max.bytes - 512 to be accepted. */ + rd_snprintf(val, sizeof(val), "%d", recv_max - 512); + test_conf_set(conf, "fetch.max.bytes", val); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Poll: the small record should arrive; fetching the oversized + * batch trips the receive cap and surfaces _BAD_MSG via error_cb. */ + for (call = 0; + call < 40 && rd_atomic32_get(&recv_max_bad_msg_seen) == 0; + call++) { + error = rd_kafka_share_poll(rkshare, 1000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + if (batch) { + small_consumed += (int)rd_kafka_messages_count(batch); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + + TEST_SAY("small records consumed=%d, _BAD_MSG seen=%d\n", + small_consumed, rd_atomic32_get(&recv_max_bad_msg_seen)); + TEST_ASSERT(small_consumed == 1, + "expected the small record to be consumed, got %d", + small_consumed); + TEST_ASSERT(rd_atomic32_get(&recv_max_bad_msg_seen) >= 1, + "expected RD_KAFKA_RESP_ERR__BAD_MSG when a batch exceeds " + "receive.message.max.bytes=%d", + recv_max); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_curr->is_fatal_cb = NULL; +} + + +/** + * @brief `internal.termination.signal` makes librdkafka signal its internal + * threads on destroy so they break out of blocking syscalls quickly. + * This is generic thread-lifecycle plumbing (not share-specific), but + * verify a share consumer with the signal configured constructs, + * runs, and closes/destroys cleanly (no hang or crash on the + * signal-driven teardown path). + * + * The signal must be handled (or ignored) by the application; we set it to + * SIG_IGN here as the test framework does for its own clients. Skipped on + * platforms without SIGIO. + */ +static void test_internal_termination_signal_destroy(void) { +#ifdef SIGIO + const char *topic; + const char *group = "0180-term-signal"; + rd_kafka_conf_t *conf; + rd_kafka_t *producer; + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + char errstr[512]; + char sigbuf[32]; + int i; + + SUB_TEST(); + + producer = create_no_linger_producer(); + topic = test_mk_topic_name("0180-term-signal", 1); + test_create_topic_wait_exists(NULL, topic, 1, -1, 60 * 1000); + produce_one_per_batch(producer, topic, 5, 100); + rd_kafka_destroy(producer); + + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "group.id", group); + /* Explicitly configure the termination signal (test_conf_init also + * sets it, but make the intent of this test self-evident). */ + rd_snprintf(sigbuf, sizeof(sigbuf), "%d", SIGIO); + test_conf_set(conf, "internal.termination.signal", sigbuf); + signal(SIGIO, SIG_IGN); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Poll a few times so the internal threads are connected and + * running (blocked in syscalls) when we tear down -- this is the + * case the termination signal is meant to short-circuit. */ + for (i = 0; i < 5; i++) { + rd_kafka_error_t *error = + rd_kafka_share_poll(rkshare, 1000, &batch); + if (error) + rd_kafka_error_destroy(error); + if (batch) { + rd_kafka_messages_destroy(batch); + batch = NULL; + } + } + + /* The actual assertion is that these complete without hanging or + * crashing on the signal-driven teardown path. */ + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + SUB_TEST_PASS(); +#else + SUB_TEST_SKIP("SIGIO not available on this platform\n"); +#endif +} + + +/* Behavioural tests that require a real broker. */ +int main_0180_share_consumer_config(int argc, char **argv) { + test_timeout_set(120); + test_max_poll_records_caps_batch_at_5(); + test_max_poll_records_allows_full_drain_at_10(); + test_fetch_max_bytes_one_is_soft_limit(); + test_fetch_min_bytes_zero_drains_all(); + test_fetch_min_bytes_max_drains_all(); + test_fetch_max_bytes_zero_returns_no_records(); + test_receive_message_max_bytes_is_honored(); + test_internal_termination_signal_destroy(); + test_share_consumer_fetch_conn_idle_after_drain(); + return 0; +} + + +/* Construction-time conf-rejection tests; no broker required. */ +int main_0180_share_consumer_config_local(int argc, char **argv) { + test_rebalance_cb_rejected_at_construction(); + test_event_rebalance_rejected_at_construction(); + test_offset_commit_cb_rejected_at_construction(); + test_consume_cb_rejected_at_construction(); + test_interceptors_rejected_for_share_consumer(); + test_enable_auto_commit_rejected_at_construction(); + test_group_protocol_rejected_at_construction(); + test_socket_max_fails_rejected_at_construction(); + test_auto_offset_reset_rejected_at_construction(); + test_consume_callback_max_messages_rejected_at_construction(); + test_session_timeout_ms_rejected_at_construction(); + test_partition_assignment_strategy_rejected_at_construction(); + test_group_protocol_type_rejected_at_construction(); + test_heartbeat_interval_ms_rejected_at_construction(); + test_group_instance_id_rejected_at_construction(); + test_isolation_level_rejected_at_construction(); + test_group_remote_assignor_rejected_at_construction(); + test_queued_min_messages_rejected_at_construction(); + test_queued_max_messages_kbytes_rejected_at_construction(); + test_fetch_queue_backoff_ms_rejected_at_construction(); + test_fetch_error_backoff_ms_rejected_at_construction(); + test_enable_partition_eof_rejected_at_construction(); + test_message_copy_max_bytes_rejected_at_construction(); + test_topic_blacklist_rejected_at_construction(); + test_receive_message_max_bytes_below_floor_rejected_at_construction(); + test_fetch_min_bytes_regular_consumer_range_rejected(); + test_reconnect_backoff_max_below_min_rejected_at_construction(); + return 0; +} \ No newline at end of file diff --git a/lib/librdkafka-2.15.0/tests/0181-share_consumer_topic_delete.c b/lib/librdkafka-2.15.0/tests/0181-share_consumer_topic_delete.c new file mode 100644 index 00000000000..9287f8d0177 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0181-share_consumer_topic_delete.c @@ -0,0 +1,354 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "testshared.h" + +/** + * @brief Share consumer topic-deletion acknowledgement tests. + * + * Consume a full batch from a 3-partition topic, then acknowledge and + * commit it either before or after deleting that topic, in implicit and + * explicit modes. The after-delete case is run with two metadata configs + * (refresh disabled vs fast refresh + no propagation delay) to exercise + * both "heartbeat reassignment first" and "metadata refresh first" + * orderings. The ordering is best-effort — the heartbeat may still win in + * the fast-refresh config — but the app-facing result is identical: topic + * deletion does not clear the cached leader, so the acknowledgement + * reaches the broker and the per-partition commit result is + * UNKNOWN_TOPIC_OR_PART / UNKNOWN_TOPIC_ID. Each case then produces to and + * drains a second, non-deleted topic to confirm the consumer keeps working + * and only the surviving topic is returned. + */ + +/** Common producer reused across all tests. */ +static rd_kafka_t *common_producer; + +/** Common admin client reused across all tests. */ +static rd_kafka_t *common_admin; + +#define TD_NPART 3 +#define TD_PER_PART 10 +#define TD_TOTAL (TD_NPART * TD_PER_PART) +#define TD_KEEP_MSGS 5 + +typedef enum { + ACK_BEFORE_DELETE, + ACK_AFTER_HB_FIRST, + ACK_AFTER_MD_FIRST, +} ack_timing_t; + +static const char *ack_timing_name(ack_timing_t t) { + switch (t) { + case ACK_BEFORE_DELETE: + return "before-delete"; + case ACK_AFTER_HB_FIRST: + return "after-delete-heartbeat-first"; + case ACK_AFTER_MD_FIRST: + return "after-delete-metadata-first"; + } + return "?"; +} + +static rd_bool_t err_is_unknown_topic(rd_kafka_resp_err_t err) { + /* TODO KIP-932: When the whole topic is deleted the broker cannot find + * the share partition and fails the entire ShareAcknowledge with a + * top-level UNKNOWN_SERVER_ERROR (RD_KAFKA_RESP_ERR_UNKNOWN) instead of + * a per-partition UNKNOWN_TOPIC_OR_PART / UNKNOWN_TOPIC_ID. Remove the + * UNKNOWN case once the broker reports the deletion per partition. */ + return err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART || + err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID || + err == RD_KAFKA_RESP_ERR_UNKNOWN; +} + +static rd_kafka_share_t *create_topic_delete_consumer(const char *group, + const char *mode, + ack_timing_t timing) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", mode); + /* Bound the batch size; the test's batch array is sized for this. */ + test_conf_set(conf, "max.poll.records", "100"); + if (timing == ACK_AFTER_MD_FIRST) { + /* Refresh metadata quickly and mark the deleted topic + * NOTEXISTS without the propagation delay, so a metadata + * refresh is likely to land before the heartbeat + * reassignment. */ + test_conf_set(conf, "topic.metadata.refresh.interval.ms", + "200"); + test_conf_set(conf, "topic.metadata.propagation.max.ms", "0"); + } + /* Otherwise use the default metadata config: the periodic refresh + * (minutes) does not fire during this short test, so the heartbeat + * reassignment lands before any metadata refresh ("heartbeat + * first"). A large explicit interval is avoided because it would + * inflate metadata.max.age.ms past the 32-bit overflow point. */ + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + return rkshare; +} + +static void do_test_topic_delete_ack(const char *mode, ack_timing_t timing) { + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + rd_kafka_topic_partition_list_t *results = NULL; + rd_kafka_error_t *error; + rd_kafka_resp_err_t del_err; + char topic_del[512], topic_keep[512], group[160]; + char del_base[128], keep_base[128]; + char *del_topics[1]; + int part_cnt[TD_NPART] = {0}; + size_t m; + int p, i; + int total = 0; + int attempts = 20; + int keep_rcvd = 0; + int idle_polls = 0; + rd_bool_t explicit_ack = !strcmp(mode, "explicit"); + rd_bool_t after_delete = timing != ACK_BEFORE_DELETE; + + SUB_TEST("mode=%s, %s", mode, ack_timing_name(timing)); + + /* Embed mode+timing in the base name so the subtests never share a + * topic even if test_mk_topic_name repeats its random suffix; + * otherwise a sibling's leftover records bleed into this drain. */ + rd_snprintf(del_base, sizeof(del_base), "0181-td-del-%s-%s", mode, + ack_timing_name(timing)); + rd_snprintf(keep_base, sizeof(keep_base), "0181-td-keep-%s-%s", mode, + ack_timing_name(timing)); + rd_snprintf(topic_del, sizeof(topic_del), "%s", + test_mk_topic_name(del_base, 1)); + rd_snprintf(topic_keep, sizeof(topic_keep), "%s", + test_mk_topic_name(keep_base, 1)); + rd_snprintf(group, sizeof(group), "0181-td-%s-%s", mode, + ack_timing_name(timing)); + del_topics[0] = topic_del; + + test_create_topic_wait_exists(NULL, topic_del, TD_NPART, -1, 60 * 1000); + test_create_topic_wait_exists(NULL, topic_keep, 1, -1, 60 * 1000); + for (p = 0; p < TD_NPART; p++) + test_produce_msgs_simple(common_producer, topic_del, p, + TD_PER_PART); + + rkshare = create_topic_delete_consumer(group, mode, timing); + /* Configure earliest after the consumer exists but before it + * subscribes/joins, so the pre-produced records are read. */ + test_share_set_auto_offset_reset(group, "earliest"); + test_share_consumer_subscribe_multi(rkshare, 2, topic_del, topic_keep); + TEST_SAY("Subscribed to %s and %s; starting phase 1 drain\n", topic_del, + topic_keep); + + /* Phase 1: drain all produced records. The partitions live on + * different brokers and the share consumer fetches one broker per + * round, so the records arrive across several batches; each round + * also flushes the previous broker's acknowledgements. Verify all + * records and that every partition is represented. */ + while (total < TD_TOTAL && attempts-- > 0) { + size_t n; + error = rd_kafka_share_poll(rkshare, 2000, &batch); + TEST_ASSERT(!error, "share_poll failed: %s", + rd_kafka_error_string(error)); + n = rd_kafka_messages_count(batch); + TEST_SAY("Phase 1: got %" PRIusz " record(s), total %d/%d\n", n, + total + (int)n, TD_TOTAL); + for (m = 0; m < n; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + TEST_ASSERT( + !strcmp(rd_kafka_topic_name(msg->rkt), topic_del), + "Phase 1 record from unexpected topic %s", + rd_kafka_topic_name(msg->rkt)); + TEST_ASSERT( + msg->partition >= 0 && msg->partition < TD_NPART, + "Unexpected partition %" PRId32, msg->partition); + part_cnt[msg->partition]++; + if (explicit_ack) { + rd_kafka_resp_err_t ack_err = + rd_kafka_share_acknowledge(rkshare, msg); + TEST_ASSERT(!ack_err, "acknowledge failed: %s", + rd_kafka_err2str(ack_err)); + } + total++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(total == TD_TOTAL, "Expected %d records, got %d", TD_TOTAL, + total); + for (p = 0; p < TD_NPART; p++) + TEST_ASSERT(part_cnt[p] == TD_PER_PART, + "Expected %d records from partition %d, got %d", + TD_PER_PART, p, part_cnt[p]); + TEST_SAY("Phase 1 complete: drained all %d records\n", TD_TOTAL); + + if (after_delete) { + TEST_SAY("Deleting topic %s before commit\n", topic_del); + del_err = test_DeleteTopics_simple(common_admin, NULL, + del_topics, 1, NULL); + TEST_ASSERT(!del_err, "DeleteTopics failed: %s", + rd_kafka_err2str(del_err)); + /* Let the chosen ordering take effect before committing. */ + rd_sleep(1); + } + + /* Commit the acknowledgements; the per-partition results carry + * the outcome for each partition. */ + TEST_SAY("Calling commit_sync\n"); + error = rd_kafka_share_commit_sync(rkshare, 30 * 1000, &results); + TEST_SAY("commit_sync returned (error: %s)\n", + error ? rd_kafka_error_string(error) : "none"); + if (error) + rd_kafka_error_destroy(error); + TEST_ASSERT(results && results->cnt > 0, + "Expected per-partition commit results"); + for (i = 0; i < results->cnt; i++) { + rd_kafka_resp_err_t perr = results->elems[i].err; + if (after_delete) + TEST_ASSERT(err_is_unknown_topic(perr), + "Expected a deleted-topic error for %s " + "[%" PRId32 "], got %s", + results->elems[i].topic, + results->elems[i].partition, + rd_kafka_err2name(perr)); + else + TEST_ASSERT(perr == RD_KAFKA_RESP_ERR_NO_ERROR, + "Expected success for %s [%" PRId32 + "], got %s", + results->elems[i].topic, + results->elems[i].partition, + rd_kafka_err2name(perr)); + } + TEST_SAY("commit_sync results verified (%d partition(s))\n", + results->cnt); + rd_kafka_topic_partition_list_destroy(results); + + if (!after_delete) { + TEST_SAY("Deleting topic %s after commit\n", topic_del); + del_err = test_DeleteTopics_simple(common_admin, NULL, + del_topics, 1, NULL); + TEST_ASSERT(!del_err, "DeleteTopics failed: %s", + rd_kafka_err2str(del_err)); + } + + /* Let the topic delete propagate through the cluster and the + * share session reconcile (the consumer subscribed to both + * topics; the broker has to drop the deleted topic from the + * session before it will start serving the surviving one + * again). Without this, the produce + consume_batch loop below + * races the session-recovery window and Phase 2 drains 0 + * records. */ + rd_sleep(3); + + /* Phase 2: drain the surviving topic to empty and verify only its + * records are returned (the deleted topic must contribute none). */ + test_produce_msgs_simple(common_producer, topic_keep, 0, TD_KEEP_MSGS); + TEST_SAY("Phase 2: draining surviving topic %s\n", topic_keep); + + /* TODO KIP-932: the deleted topic makes the broker fail the whole + * session with a top-level UNKNOWN for a while, so we give the drain + * a larger budget and ignore empty polls until all records arrive. + * Revert to a plain drain loop once the broker is fixed. */ + attempts = 60; + while (attempts-- > 0) { + size_t got; + error = rd_kafka_share_poll(rkshare, 1000, &batch); + TEST_ASSERT(!error, "share_poll failed: %s", + rd_kafka_error_string(error)); + got = rd_kafka_messages_count(batch); + TEST_SAY("Phase 2: got %" PRIusz + " record(s), kept %d/%d (idle %d)\n", + got, keep_rcvd, TD_KEEP_MSGS, idle_polls); + if (got == 0) { + /* An empty poll counts as drained only after every + * record has arrived; earlier ones are the session + * still recovering. */ + rd_kafka_messages_destroy(batch); + batch = NULL; + if (keep_rcvd >= TD_KEEP_MSGS && ++idle_polls >= 3) + break; + continue; + } + idle_polls = 0; + for (m = 0; m < got; m++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, m); + TEST_ASSERT( + !strcmp(rd_kafka_topic_name(msg->rkt), topic_keep), + "Phase 2 returned a record from %s; only %s " + "should remain", + rd_kafka_topic_name(msg->rkt), topic_keep); + if (explicit_ack) + rd_kafka_share_acknowledge(rkshare, msg); + keep_rcvd++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(keep_rcvd == TD_KEEP_MSGS, + "Expected to drain exactly %d records from surviving " + "topic %s, got %d", + TD_KEEP_MSGS, topic_keep, keep_rcvd); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + /* TODO KIP-932: add a delete-then-recreate variant once topic + * recreation handling is fixed. */ + SUB_TEST_PASS(); +} + +int main_0181_share_consumer_topic_delete(int argc, char **argv) { + /* Topic deletion is not supported against Windows brokers. */ + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) { + TEST_SKIP("Topic deletion not supported on Windows brokers\n"); + return 0; + } + + /* Budget for all subtests (a timeout multiplier is applied on top). */ + test_timeout_set(120); + + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + do_test_topic_delete_ack("implicit", ACK_BEFORE_DELETE); + do_test_topic_delete_ack("implicit", ACK_AFTER_HB_FIRST); + do_test_topic_delete_ack("implicit", ACK_AFTER_MD_FIRST); + do_test_topic_delete_ack("explicit", ACK_BEFORE_DELETE); + do_test_topic_delete_ack("explicit", ACK_AFTER_HB_FIRST); + do_test_topic_delete_ack("explicit", ACK_AFTER_MD_FIRST); + + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0182-share_consumer_error_handling_mock.c b/lib/librdkafka-2.15.0/tests/0182-share_consumer_error_handling_mock.c new file mode 100644 index 00000000000..d7b80e401d2 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0182-share_consumer_error_handling_mock.c @@ -0,0 +1,3358 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @brief Share consumer top-level error propagation through ShareAcknowledge. + * + * Verifies that a top-level err on a ShareAcknowledge response reaches + * the per-partition rktpar->err that commit_sync reads, with no + * _IN_PROGRESS sentinel leaking to the caller. The broker-thread helper + * (rd_kafka_share_fetch_op_reply_with_err) sets err on each batch in + * ack_details; the main reply handler copies batch->rktpar->err into + * rkcg_commit_sync_request.results. + * + * Out of scope here (need mock enhancements not yet available): + * - Per-partition AcknowledgementErrorCode injection (mock only supports + * top-level err push) + * - Partition missing from ShareAcknowledge response (no mock API to + * drop a partition from response) + */ + +#define CONSUME_ARRAY 1024 + +/* =================================================================== + * Mock broker infrastructure (same pattern as 0176). + * =================================================================== */ +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new_n(int nbrok) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(nbrok, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + return ctx; +} + +static test_ctx_t test_ctx_new(void) { + return test_ctx_new_n(1); +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static rd_kafka_share_t * +create_mock_share_consumer(const char *bootstraps, + const char *group_id, + const char *ack_mode, + test_ack_cb_state_t *cb_state, + void (*cb)(rd_kafka_share_t *, + rd_kafka_share_partition_offsets_list_t *, + rd_kafka_resp_err_t, + void *)) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "share.acknowledgement.mode", ack_mode); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + /* Register acknowledgement callback at runtime */ + if (cb && cb_state) { + rd_kafka_error_t *error = + rd_kafka_share_set_acknowledgement_commit_cb(rkshare, cb, + cb_state); + TEST_ASSERT(error == NULL, + "Failed to set acknowledgement commit callback: " + "%s", + rd_kafka_error_string(error)); + } + return rkshare; +} + +static void mock_produce(rd_kafka_t *producer, const char *topic, int msgcnt) { + int i; + for (i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "%s-%d", topic, i); + TEST_ASSERT(rd_kafka_producev( + producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Produce failed"); + } + rd_kafka_flush(producer, 5000); +} + +static void mock_produce_partition(rd_kafka_t *producer, + const char *topic, + int32_t partition, + int msgcnt) { + int i; + for (i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "%s-p%d-%d", topic, + (int)partition, i); + TEST_ASSERT(rd_kafka_producev( + producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_PARTITION(partition), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Produce to partition %d failed", (int)partition); + } + rd_kafka_flush(producer, 5000); +} + +/** + * @brief Consume up to msgcnt records and ACCEPT each, returning count + * actually acknowledged. + */ +static int consume_and_ack_all(rd_kafka_share_t *rkshare, int msgcnt) { + rd_kafka_messages_t *batch = NULL; + int acked = 0; + int attempts = 0; + + while (acked < msgcnt && attempts++ < 30) { + size_t rcvd = 0; + size_t j; + rd_kafka_error_t *error = + rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + acked++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + return acked; +} + +/* =================================================================== + * Parameterized helper: inject one top-level err on next + * ShareAcknowledge response and verify commit_sync result carries + * that err on every partition. + * + * This exercises the broker-thread helper + * (rd_kafka_share_fetch_op_reply_with_err) + the main reply handler + * defensive _IN_PROGRESS sentinel together: regardless of which + * layer writes batch->rktpar->err, the commit_sync caller must see + * the top-level err for every partition that was sent to the broker. + * =================================================================== */ +static void +do_test_commit_sync_top_level_err(const char *test_name, + rd_kafka_resp_err_t injected_err) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + char topic[64]; + char group[64]; + const int msgcnt = 10; + int acked; + int i; + test_ack_cb_state_t cb_state = {0}; + + SUB_TEST_QUICK("%s -> %s", test_name, rd_kafka_err2name(injected_err)); + + ctx = test_ctx_new(); + + rd_snprintf(topic, sizeof(topic), "0182-%s", test_name); + rd_snprintf(group, sizeof(group), "sg-0182-%s", test_name); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + &cb_state, test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + acked = consume_and_ack_all(rkshare, msgcnt); + TEST_ASSERT(acked == msgcnt, "expected %d acked, got %d", msgcnt, + acked); + + /* Inject the top-level err on the next ShareAcknowledge + * response from broker 1 (only broker in cluster). */ + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + injected_err, 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push error"); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "expected non-NULL partition results " + "(top-level err must surface per-partition, " + "not as _IN_PROGRESS leak)"); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("%s [%" PRId32 "]: %s\n", rktpar->topic, + rktpar->partition, rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == injected_err, "expected %s, got %s", + rd_kafka_err2name(injected_err), + rd_kafka_err2name(rktpar->err)); + } + + rd_kafka_topic_partition_list_destroy(partitions); + + /* Verify callback was invoked with the error */ + TEST_ASSERT(cb_state.callback_cnt == 1, "expected 1 callback, got %d", + cb_state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == injected_err, + "expected callback err %s, got %s", + rd_kafka_err2name(injected_err), + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + TEST_ASSERT(cb_state.total_offsets == msgcnt, + "expected callback total_offsets %d, got %zu", msgcnt, + cb_state.total_offsets); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&cb_state); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/* Top-level session error: SHARE_SESSION_NOT_FOUND on ShareAcknowledge + * is propagated to every partition in commit_sync results (no + * _IN_PROGRESS leak). */ +static void test_commit_sync_share_session_not_found(void) { + do_test_commit_sync_top_level_err( + "session-not-found", RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND); +} + +/* Same path with a different session error code — confirms + * propagation isn't tied to a specific err. */ +static void test_commit_sync_invalid_share_session_epoch(void) { + do_test_commit_sync_top_level_err( + "invalid-session-epoch", + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH); +} + +/* commit_sync surfaces a top-level SHARE_SESSION_LIMIT_REACHED from + * ShareAcknowledge without any client-side special handling. */ +static void test_commit_sync_share_session_limit_reached(void) { + do_test_commit_sync_top_level_err( + "session-limit-reached", + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED); +} + +/* GROUP_AUTHORIZATION_FAILED on ShareAcknowledge propagates through + * the default-case path (previously hit `default: break` with no ack + * propagation). */ +static void test_commit_sync_group_authorization_failed(void) { + do_test_commit_sync_top_level_err( + "group-auth-failed", RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED); +} + +/* Same default-case path as group-auth-failed with a different err. */ +static void test_commit_sync_topic_authorization_failed(void) { + do_test_commit_sync_top_level_err( + "topic-auth-failed", RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); +} + +/* Generic protocol error to confirm unknown / fatal codes also + * propagate through the default-case path. */ +static void test_commit_sync_invalid_request(void) { + do_test_commit_sync_top_level_err("invalid-request", + RD_KAFKA_RESP_ERR_INVALID_REQUEST); +} + +/* =================================================================== + * Test — top-level error is propagated to all partitions in + * multi-partition acknowledgement. + * + * Creates a topic with multiple partitions, consumes and acknowledges + * records from all partitions, injects a top-level error on + * ShareAcknowledge, and verifies that commit_sync returns the error + * for EVERY partition that was part of the acknowledgement request. + * + * This ensures the error propagation logic correctly applies the + * top-level error to all partitions in the ack_details list, not + * just the first one. + * =================================================================== */ +static void test_commit_sync_multi_partition_top_level_error(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + const char *topic = "0182-multi-partition-error"; + const char *group = "sg-0182-multi-partition-error"; + const int partition_cnt = 3; + const int msgs_per_partition = 5; + const int total_msgs = partition_cnt * msgs_per_partition; + rd_kafka_resp_err_t injected_err = + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH; + rd_kafka_messages_t *batches[CONSUME_ARRAY] = {0}; + int batch_cnt = 0; + int total_consumed = 0; + int acked = 0; + int attempts = 0; + int i; + test_ack_cb_state_t cb_state = {0}; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + /* Create topic with multiple partitions */ + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, + partition_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic with %d partitions", partition_cnt); + + /* Produce messages to all partitions explicitly */ + for (i = 0; i < partition_cnt; i++) + mock_produce_partition(ctx.producer, topic, i, + msgs_per_partition); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + &cb_state, test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Consume messages from all partitions */ + while (total_consumed < total_msgs && attempts++ < 50) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + continue; + } + rcvd = rd_kafka_messages_count(batch); + if (rcvd == 0) { + rd_kafka_messages_destroy(batch); + continue; + } + TEST_ASSERT(batch_cnt < CONSUME_ARRAY, "batch buffer overflow"); + batches[batch_cnt++] = batch; + total_consumed += (int)rcvd; + } + + TEST_ASSERT(total_consumed == total_msgs, + "expected to consume %d messages from %d partitions, " + "got %d", + total_msgs, partition_cnt, total_consumed); + + /* Acknowledge all messages */ + for (i = 0; i < batch_cnt; i++) { + size_t j; + size_t rcvd = rd_kafka_messages_count(batches[i]); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batches[i], j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + acked++; + } + } + } + + TEST_ASSERT(acked == total_msgs, "expected to ack %d messages, got %d", + total_msgs, acked); + + /* Inject top-level error on next ShareAcknowledge */ + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + injected_err, 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push error"); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, "expected non-NULL partition results"); + + /* Verify ALL partitions have the injected error */ + TEST_ASSERT(partitions->cnt == partition_cnt, + "expected results for %d partitions, got %d", partition_cnt, + partitions->cnt); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("%s [%" PRId32 "]: %s\n", rktpar->topic, + rktpar->partition, rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == injected_err, + "partition [%" PRId32 "]: expected %s, got %s", + rktpar->partition, rd_kafka_err2name(injected_err), + rd_kafka_err2name(rktpar->err)); + } + + rd_kafka_topic_partition_list_destroy(partitions); + + /* Verify callback was invoked with the error. + * Callback is invoked once per partition, so we expect + * partition_cnt callbacks. */ + TEST_ASSERT(cb_state.callback_cnt == partition_cnt, + "expected %d callbacks (one per partition), got %d", + partition_cnt, cb_state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == injected_err, + "expected callback err %s, got %s", + rd_kafka_err2name(injected_err), + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + TEST_ASSERT(cb_state.total_offsets == total_msgs, + "expected callback total_offsets %d, got %zu", total_msgs, + cb_state.total_offsets); + + for (i = 0; i < batch_cnt; i++) + rd_kafka_messages_destroy(batches[i]); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&cb_state); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * Test — top-level error on ShareFetch with piggybacked acks is + * propagated to callback for all partitions. + * + * Uses implicit mode where acks are piggybacked on ShareFetch. + * Creates a topic with multiple partitions, consumes messages in + * implicit mode (auto-acknowledge), then injects a top-level error + * on the next ShareFetch request (which carries piggybacked acks). + * + * Verifies that the acknowledgement callback is invoked with the + * error for all partitions that had piggybacked acks. + * =================================================================== */ +static void test_consume_batch_multi_partition_top_level_error(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + const char *topic = "0182-consume-multi-partition-error"; + const char *group = "sg-0182-consume-multi-partition-error"; + const int partition_cnt = 3; + const int msgs_per_partition = 5; + const int total_msgs = partition_cnt * msgs_per_partition; + rd_kafka_resp_err_t injected_err = + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND; + rd_kafka_messages_t *batch = NULL; + int total_consumed = 0; + int attempts = 0; + int i; + test_ack_cb_state_t cb_state = {0}; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + /* Create topic with multiple partitions */ + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, + partition_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic with %d partitions", partition_cnt); + + /* Produce messages to all partitions explicitly */ + for (i = 0; i < partition_cnt; i++) + mock_produce_partition(ctx.producer, topic, i, + msgs_per_partition); + + /* Use implicit mode - acks are piggybacked on ShareFetch */ + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "implicit", + &cb_state, test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* First consume batch - establishes session, consumes messages */ + while (total_consumed < total_msgs && attempts++ < 50) { + size_t rcvd = 0; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + rcvd = rd_kafka_messages_count(batch); + total_consumed += (int)rcvd; + /* Destroy messages - in implicit mode they're + * auto-acknowledged */ + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(total_consumed == total_msgs, + "expected to consume %d messages from %d partitions, " + "got %d", + total_msgs, partition_cnt, total_consumed); + + /* Produce more messages to trigger another ShareFetch with + * piggybacked acks from the previous consume */ + for (i = 0; i < partition_cnt; i++) + mock_produce_partition(ctx.producer, topic, i, + msgs_per_partition); + + /* Inject top-level error on next ShareFetch (which will carry + * piggybacked acks from the previous consume_batch) */ + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareFetch, 1, injected_err, + 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push error on ShareFetch"); + + /* Next consume_batch triggers ShareFetch with piggybacked acks. + * The error should trigger the callback for the piggybacked acks. */ + total_consumed = 0; + attempts = 0; + while (total_consumed < total_msgs && attempts++ < 50) { + size_t rcvd = 0; + size_t j; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + /* May get errors due to injected error */ + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + total_consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + /* Break after we've given the callback a chance to fire */ + if (cb_state.callback_cnt > 0) + break; + } + + /* Verify callback was invoked with the error for piggybacked acks. + * The callback should have been invoked for the acks that were + * piggybacked on the ShareFetch that got the error. */ + TEST_ASSERT(cb_state.callback_cnt >= 1, + "expected at least 1 callback for piggybacked acks, got %d", + cb_state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == injected_err, + "expected callback err %s, got %s", + rd_kafka_err2name(injected_err), + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + /* In implicit mode, we expect the callback to be invoked for the + * first batch of messages that were piggybacked */ + TEST_ASSERT(cb_state.total_offsets > 0, + "expected callback total_offsets > 0, got %zu", + cb_state.total_offsets); + + TEST_SAY( + "Callback invoked %d times with %zu total offsets, first_err=%s\n", + cb_state.callback_cnt, cb_state.total_offsets, + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&cb_state); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * Test — commit_sync at session epoch 0 returns + * INVALID_SHARE_SESSION_EPOCH without sending a + * ShareAcknowledge request. + * + * When the broker session epoch is 0 (new consumer or post-reset) + * there is no session state to acknowledge against, so the client + * must fail acks locally. + * + * Two-phase test: + * Phase 1: Trigger session reset by injecting + * SHARE_SESSION_NOT_FOUND on the first ShareAcknowledge. + * commit_sync surfaces SHARE_SESSION_NOT_FOUND for the + * partition; broker thread resets epoch to 0. + * Phase 2: Acknowledge remaining records and call commit_sync + * again. With epoch 0 the client fails acks locally: no + * ShareAcknowledge is sent, commit_sync returns + * INVALID_SHARE_SESSION_EPOCH for the partition. + * =================================================================== */ +static rd_bool_t is_share_ack_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == + RD_KAFKAP_ShareAcknowledge; +} + +static void +test_commit_sync_at_epoch_zero_returns_invalid_session_epoch_error(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + const char *topic = "0182-epoch-zero-ack"; + const char *group = "sg-0182-epoch-zero-ack"; + const int msgcnt = 10; + rd_kafka_message_t *rkmessages[CONSUME_ARRAY]; + rd_kafka_messages_t *batches[CONSUME_ARRAY] = {0}; + int batch_cnt = 0; + int total_consumed = 0; + int attempts = 0; + size_t share_ack_cnt; + int i; + test_ack_cb_state_t cb_state = {0}; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + &cb_state, test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Phase 0: consume all 10 records. Hold message handles for + * acknowledge in phase 1 and phase 2. */ + while (total_consumed < msgcnt && attempts++ < 30) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + size_t j; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + continue; + } + rcvd = rd_kafka_messages_count(batch); + if (rcvd == 0) { + rd_kafka_messages_destroy(batch); + continue; + } + /* Flatten message handles into rkmessages for the + * phase-1/phase-2 partial-ack indexing pattern. */ + for (j = 0; j < rcvd && total_consumed < CONSUME_ARRAY; j++) + rkmessages[total_consumed++] = + rd_kafka_messages_get(batch, j); + TEST_ASSERT(batch_cnt < CONSUME_ARRAY, "batch buffer overflow"); + batches[batch_cnt++] = batch; + } + TEST_ASSERT(total_consumed == msgcnt, + "Phase 0: expected %d records, got %d", msgcnt, + total_consumed); + + /* Phase 1: ACCEPT first 5 records, inject SHARE_SESSION_NOT_FOUND + * on next ShareAcknowledge, call commit_sync and verify the err + * propagates. The buf reply handler resets the broker session + * epoch to 0 on this error. */ + for (i = 0; i < 5; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push SHARE_SESSION_NOT_FOUND"); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 1000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 1: expected non-NULL partition results"); + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *p = &partitions->elems[i]; + TEST_SAY("Phase 1 %s [%" PRId32 "]: %s\n", p->topic, + p->partition, rd_kafka_err2name(p->err)); + TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + "Phase 1: expected SHARE_SESSION_NOT_FOUND, " + "got %s", + rd_kafka_err2name(p->err)); + } + rd_kafka_topic_partition_list_destroy(partitions); + + share_ack_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_ack_request, NULL); + TEST_ASSERT(share_ack_cnt == 1, + "Phase 1: expected 1 ShareAck request, got %" PRIusz, + share_ack_cnt); + + /* Verify Phase 1 callback was invoked with SHARE_SESSION_NOT_FOUND */ + TEST_ASSERT(cb_state.callback_cnt == 1, + "Phase 1: expected 1 callback, got %d", + cb_state.callback_cnt); + TEST_ASSERT( + test_ack_cb_state_first_err(&cb_state) == + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + "Phase 1: expected callback err SHARE_SESSION_NOT_FOUND, got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + TEST_ASSERT(cb_state.total_offsets == 5, + "Phase 1: expected 5 offsets in callback, got %zu", + cb_state.total_offsets); + + /* Reset callback state for Phase 2 */ + test_ack_cb_state_destroy(&cb_state); + + /* Phase 2: ACCEPT remaining 5 records and call commit_sync + * again. Broker epoch is 0 (session reset by phase 1). B4a + * must fire: no ShareAck request sent, commit_sync returns + * INVALID_SHARE_SESSION_EPOCH per partition. */ + rd_kafka_mock_clear_requests(ctx.mcluster); + + for (i = 5; i < msgcnt; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 1000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 2: expected non-NULL partition results"); + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *p = &partitions->elems[i]; + TEST_SAY("Phase 2 %s [%" PRId32 "]: %s\n", p->topic, + p->partition, rd_kafka_err2name(p->err)); + TEST_ASSERT(p->err == + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH, + "Phase 2: expected INVALID_SHARE_SESSION_EPOCH, " + "got %s", + rd_kafka_err2name(p->err)); + } + rd_kafka_topic_partition_list_destroy(partitions); + + share_ack_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_ack_request, NULL); + TEST_ASSERT(share_ack_cnt == 0, + "Phase 2: expected 0 ShareAck requests " + "(local epoch-0 fail should have prevented send), " + "got %" PRIusz, + share_ack_cnt); + + /* Verify Phase 2 callback was invoked with INVALID_SHARE_SESSION_EPOCH + * (local fail because session epoch is 0) */ + TEST_ASSERT(cb_state.callback_cnt == 1, + "Phase 2: expected 1 callback, got %d", + cb_state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH, + "Phase 2: expected callback err " + "INVALID_SHARE_SESSION_EPOCH, got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + TEST_ASSERT(cb_state.total_offsets == 5, + "Phase 2: expected 5 offsets in callback, got %zu", + cb_state.total_offsets); + + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + for (i = 0; i < batch_cnt; i++) + rd_kafka_messages_destroy(batches[i]); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&cb_state); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * Test — consume_batch with session epoch 0 strips piggybacked + * acks from the ShareFetch wire request. + * + * Same shape as test_commit_sync_at_epoch_zero_..._error but Phase + * 2 calls consume_batch (which triggers a FANOUT -> ShareFetch + * with should_fetch=true) instead of commit_sync. With session + * epoch == 0 and any cached piggyback acks attached, the strip + * path in broker_share_rpc fires: acks are pre-set with + * INVALID_SHARE_SESSION_EPOCH and detached from rko before + * ShareFetch is built, so the wire request goes out with no ack + * data section. + * + * Asserts (per-partition): + * - Wire-level: ShareFetch is sent (>= 1), no extra ShareAck + * requests fire. + * - Acknowledgement callback fires with + * INVALID_SHARE_SESSION_EPOCH for each stripped batch (the + * session-establish ShareFetch's response would normally cause + * the parser to write the broker's per-partition + * AcknowledgementErrorCode, but the parser conditional preserves + * our pre-set). + * + * To verify the strip path manually, run with: + * TEST_DEBUG=fetch,broker,cgrp TESTS=0182 \ + * SUBTESTS=test_consume_batch_at_epoch_zero make + * and look for the "Stripping N piggybacked ack batches" SHAREFETCH + * log line in stderr. + * =================================================================== */ +static rd_bool_t is_share_fetch_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == RD_KAFKAP_ShareFetch; +} + +static void test_consume_batch_at_epoch_zero_strips_piggyback_acks(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + const char *topic = "0182-epoch-zero-piggyback"; + const char *group = "sg-0182-epoch-zero-piggyback"; + const int msgcnt = 10; + rd_kafka_message_t *rkmessages[CONSUME_ARRAY]; + rd_kafka_messages_t *phase0_batch = NULL; + rd_kafka_messages_t *phase2_batch = NULL; + int attempts = 0; + size_t rcvd = 0; + size_t share_ack_cnt; + size_t share_fetch_cnt; + int i; + test_ack_cb_state_t cb_state = {0}; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + &cb_state, test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Phase 0: consume all msgcnt records in a single consume_batch + * call (with retry-on-empty for transient cases). Explicit-mode + * contract requires every record from a previous consume_batch + * to be acknowledged before the next call, so we must not loop + * and accumulate without acking each batch first. msgcnt is far + * below max.poll.records (default 500), so a non-empty call + * returns the full set. */ + while (rcvd == 0 && attempts++ < 30) { + rd_kafka_messages_destroy(phase0_batch); + phase0_batch = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &phase0_batch); + if (error) + rd_kafka_error_destroy(error); + rcvd = rd_kafka_messages_count(phase0_batch); + } + TEST_ASSERT(rcvd == (size_t)msgcnt, + "Phase 0: expected %d records in single batch, " + "got %" PRIusz, + msgcnt, rcvd); + for (i = 0; i < msgcnt; i++) + rkmessages[i] = rd_kafka_messages_get(phase0_batch, i); + + /* Phase 1: ACCEPT first 5 records, inject SHARE_SESSION_NOT_FOUND + * on next ShareAcknowledge, call commit_sync to trigger session + * reset on the broker (epoch -> 0). */ + for (i = 0; i < 5; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push SHARE_SESSION_NOT_FOUND"); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 1000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 1: expected non-NULL partition results"); + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *p = &partitions->elems[i]; + TEST_SAY("Phase 1 %s [%" PRId32 "]: %s\n", p->topic, + p->partition, rd_kafka_err2name(p->err)); + TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + "Phase 1: expected SHARE_SESSION_NOT_FOUND, " + "got %s", + rd_kafka_err2name(p->err)); + } + rd_kafka_topic_partition_list_destroy(partitions); + + share_ack_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_ack_request, NULL); + TEST_ASSERT(share_ack_cnt == 1, + "Phase 1: expected 1 ShareAck request, got %" PRIusz, + share_ack_cnt); + + /* Verify Phase 1 callback was invoked with SHARE_SESSION_NOT_FOUND */ + TEST_ASSERT(cb_state.callback_cnt == 1, + "Phase 1: expected 1 callback, got %d", + cb_state.callback_cnt); + TEST_ASSERT( + test_ack_cb_state_first_err(&cb_state) == + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + "Phase 1: expected callback err SHARE_SESSION_NOT_FOUND, got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + + /* Reset callback state for Phase 2 */ + test_ack_cb_state_destroy(&cb_state); + + /* Phase 2: ACCEPT remaining 5 records and call consume_batch + * (NOT commit_sync). This triggers a FANOUT -> ShareFetch with + * should_fetch=true. Broker epoch is 0 (reset in phase 1). If + * any piggyback acks are attached when broker_share_rpc runs, + * the strip path fires and pre-sets each batch's err to + * INVALID_SHARE_SESSION_EPOCH. ShareFetch goes out with no ack + * data section. + * + * Wire-level assertions: + * - At least 1 new ShareFetch request was sent (session + * re-establishment). + * - 0 new ShareAck requests (strip prevented send if any + * piggyback acks were present; ack-only path also doesn't + * fire because we did not call commit_sync). */ + rd_kafka_mock_clear_requests(ctx.mcluster); + + for (i = 5; i < msgcnt; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + /* Single consume_batch triggers the strip-mode FANOUT. + * Records here may be re-deliveries after lock expiry — we + * don't assert on contents, only on wire-level counts. + * test_wait_for_cb_with_poll below tolerates the explicit-mode + * __STATE the next consume_batch may return for these + * un-acknowledged records (rcvd stays 0). */ + error = rd_kafka_share_poll(rkshare, 1000, &phase2_batch); + if (error) + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(phase2_batch); + phase2_batch = NULL; + + share_fetch_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_fetch_request, NULL); + share_ack_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_ack_request, NULL); + + TEST_SAY("Phase 2 wire counts: ShareFetch=%" PRIusz + ", ShareAck=%" PRIusz "\n", + share_fetch_cnt, share_ack_cnt); + + TEST_ASSERT(share_fetch_cnt >= 1, + "Phase 2: expected >= 1 ShareFetch (session " + "re-establish), got %" PRIusz, + share_fetch_cnt); + TEST_ASSERT(share_ack_cnt == 0, + "Phase 2: expected 0 ShareAck " + "(strip should have prevented any piggyback ack " + "send and no ack-only path fired), got %" PRIusz, + share_ack_cnt); + + /* Wait for the per-partition acknowledgement callback to fire + * (dispatched on rk_rep when the SHARE_FETCH op reply reaches + * the main thread). */ + TEST_ASSERT(test_wait_for_cb_with_poll(&cb_state, rkshare, 1, 5000), + "Phase 2: timed out waiting for ack callback"); + + /* The strip path pre-set INVALID_SHARE_SESSION_EPOCH on each + * batch in ack_details. The session-establish ShareFetch + * succeeds and the broker echoes the added partition in its + * response, so the parser would normally write the broker's + * AcknowledgementErrorCode (NO_ERROR) onto the batch — the + * parser conditional (override only _IN_PROGRESS) preserves + * the pre-set instead. The callback fires with the pre-set + * err. */ + TEST_ASSERT(cb_state.callback_cnt == 1, + "Phase 2: expected 1 callback, got %d", + cb_state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH, + "Phase 2: expected callback err " + "INVALID_SHARE_SESSION_EPOCH, got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + TEST_ASSERT(cb_state.total_offsets == 5, + "Phase 2: expected 5 acked offsets in callback, " + "got %" PRIusz, + cb_state.total_offsets); + + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + rd_kafka_messages_destroy(phase0_batch); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&cb_state); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * Test — strip + ShareFetch response top-level err: callback err + * remains INVALID_SHARE_SESSION_EPOCH (not the response err). + * + * Same flow as test_consume_batch_at_epoch_zero_strips_piggyback_acks + * but in Phase 2 we ALSO inject a top-level err on the + * session-establish ShareFetch response. The buf-callback's helper + * (rd_kafka_share_fetch_op_reply_with_err) will be called with that + * err — its conditional override (only _IN_PROGRESS) must NOT + * clobber the pre-set INVALID_SHARE_SESSION_EPOCH on the stripped + * batches. + * + * Asserts (per-partition via callback): + * - Wire-level: ShareFetch is sent, no ShareAck. + * - Acknowledgement callback fires with + * INVALID_SHARE_SESSION_EPOCH (NOT the injected response err) + * for each stripped batch. + * =================================================================== */ +static void test_strip_pre_set_survives_sharefetch_err(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + const char *topic = "0182-epoch-zero-piggyback-fetch-err"; + const char *group = "sg-0182-epoch-zero-piggyback-fetch-err"; + const int msgcnt = 10; + rd_kafka_message_t *rkmessages[CONSUME_ARRAY]; + rd_kafka_messages_t *phase0_batch = NULL; + rd_kafka_messages_t *phase2_batch = NULL; + int attempts = 0; + size_t rcvd = 0; + size_t share_ack_cnt; + size_t share_fetch_cnt; + int i; + test_ack_cb_state_t cb_state = {0}; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce(ctx.producer, topic, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + &cb_state, test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Phase 0: consume all msgcnt records in a single consume_batch + * call (with retry-on-empty for transient cases). Explicit-mode + * contract requires every record from a previous consume_batch + * to be acknowledged before the next call, so we must not loop + * and accumulate without acking each batch first. msgcnt is far + * below max.poll.records (default 500), so a non-empty call + * returns the full set. */ + while (rcvd == 0 && attempts++ < 30) { + rd_kafka_messages_destroy(phase0_batch); + phase0_batch = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &phase0_batch); + if (error) + rd_kafka_error_destroy(error); + rcvd = rd_kafka_messages_count(phase0_batch); + } + TEST_ASSERT(rcvd == (size_t)msgcnt, + "Phase 0: expected %d records in single batch, " + "got %" PRIusz, + msgcnt, rcvd); + for (i = 0; i < msgcnt; i++) + rkmessages[i] = rd_kafka_messages_get(phase0_batch, i); + + /* Phase 1: ACCEPT first 5 records, inject SHARE_SESSION_NOT_FOUND + * on next ShareAcknowledge, call commit_sync to trigger session + * reset on the broker (epoch -> 0). */ + for (i = 0; i < 5; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push SHARE_SESSION_NOT_FOUND"); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 1000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 1: expected non-NULL partition results"); + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *p = &partitions->elems[i]; + TEST_ASSERT(p->err == RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND, + "Phase 1: expected SHARE_SESSION_NOT_FOUND, " + "got %s", + rd_kafka_err2name(p->err)); + } + rd_kafka_topic_partition_list_destroy(partitions); + + /* Reset callback state for Phase 2 */ + test_ack_cb_state_destroy(&cb_state); + + /* Phase 2: ACCEPT remaining 5 records. Inject + * SHARE_SESSION_LIMIT_REACHED on the next ShareFetch — this is + * the strip-mode ShareFetch (epoch 0, with stripped piggyback + * acks) that goes out from consume_batch. The buf-callback's + * helper is then called with SHARE_SESSION_LIMIT_REACHED on a + * batch already pre-set to INVALID_SHARE_SESSION_EPOCH; the + * helper conditional (only override _IN_PROGRESS) must + * preserve the pre-set. */ + rd_kafka_mock_clear_requests(ctx.mcluster); + + for (i = 5; i < msgcnt; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + ctx.mcluster, 1, RD_KAFKAP_ShareFetch, 1, + RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED, + 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push SHARE_SESSION_LIMIT_REACHED on next ShareFetch"); + + /* Single consume_batch triggers the strip-mode FANOUT. + * Records here may be re-deliveries after lock expiry — we + * don't assert on contents, only on wire-level counts. + * test_wait_for_cb_with_poll below tolerates the explicit-mode + * __STATE the next consume_batch may return for these + * un-acknowledged records (rcvd stays 0). */ + error = rd_kafka_share_poll(rkshare, 1000, &phase2_batch); + if (error) + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(phase2_batch); + phase2_batch = NULL; + + share_fetch_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_fetch_request, NULL); + share_ack_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_ack_request, NULL); + + TEST_SAY("Phase 2 wire counts: ShareFetch=%" PRIusz + ", ShareAck=%" PRIusz "\n", + share_fetch_cnt, share_ack_cnt); + + TEST_ASSERT(share_fetch_cnt >= 1, + "Phase 2: expected >= 1 ShareFetch, got %" PRIusz, + share_fetch_cnt); + TEST_ASSERT(share_ack_cnt == 0, + "Phase 2: expected 0 ShareAck (strip), got %" PRIusz, + share_ack_cnt); + + /* Wait for the per-partition acknowledgement callback. */ + TEST_ASSERT(test_wait_for_cb_with_poll(&cb_state, rkshare, 1, 5000), + "Phase 2: timed out waiting for ack callback"); + + /* The injected ShareFetch top-level err + * (SHARE_SESSION_LIMIT_REACHED) reached the + * rd_kafka_share_fetch_op_reply_with_err helper. The helper's + * conditional override (only _IN_PROGRESS) must NOT clobber + * the pre-set INVALID_SHARE_SESSION_EPOCH. */ + TEST_ASSERT(cb_state.callback_cnt == 1, + "Phase 2: expected 1 callback, got %d", + cb_state.callback_cnt); + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == + RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH, + "Phase 2: expected callback err " + "INVALID_SHARE_SESSION_EPOCH (pre-set preserved), " + "got %s", + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + TEST_ASSERT(cb_state.total_offsets == 5, + "Phase 2: expected 5 acked offsets in callback, " + "got %" PRIusz, + cb_state.total_offsets); + + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + rd_kafka_messages_destroy(phase0_batch); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ack_cb_state_destroy(&cb_state); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * Wire-level socket timeout matrix. + * + * Exercises commit_sync under a socket-timeout-induced connection + * teardown. The two timer layers in the share-acknowledge path are + * intentionally decoupled: + * + * - The app-visible commit_sync(timeout_ms) deadline drives the + * timeout cb that stamps REQUEST_TIMED_OUT on the per-partition + * results. + * - The wire RPC carries the broker connection's socket.timeout.ms + * and tears the connection down (default socket.max.fails = 1) + * when it fires. + * + * Three (api_timeout_ms, socket_timeout_ms, rtt_ms) triples cover + * the cases where api > socket, api == socket, and api < socket. In + * all three, rtt_ms > socket_timeout_ms so the wire eventually times + * out and the connection is torn down (broker drops the session as + * a result of the TCP close). + * + * Two flow variants build on the same setup: + * + * full_ack_then_more — Phase 1 commits all consumed records. + * Phase 2 produces and consumes new records on a fresh session + * and expects NO_ERROR on the second commit_sync. + * + * partial_ack_then_remaining — Phase 1 commits only half. The + * records left in the local inflight map were ACQUIRED on the + * now-dropped session, so Phase 2's commit_sync surfaces an + * INVALID_RECORD_STATE / SHARE_SESSION_NOT_FOUND / + * INVALID_SHARE_SESSION_EPOCH-class error (when session-err + * translation lands at the app-facing boundary this narrows + * to INVALID_RECORD_STATE). + * + * TODO KIP-932: add multi-broker variants once the single-broker + * matrix is stable. Multi-broker exercises the case where only one + * broker is slow — partitions on other brokers should commit + * immediately without waiting for the slow broker's socket timer. + * =================================================================== */ + +#define SOCKET_TIMEOUT_MATRIX_RECORD_LOCK_MS 600000 +#define SOCKET_TIMEOUT_MATRIX_PARTITIONS 3 +#define SOCKET_TIMEOUT_MATRIX_MSGS_PER_PARTITION 10 + +/** + * @brief Create a share consumer with a custom socket.timeout.ms. + */ +static rd_kafka_share_t *create_share_consumer_socket_timeout( + const char *bootstraps, + const char *group_id, + const char *ack_mode, + int socket_timeout_ms, + test_ack_cb_state_t *cb_state, + void (*cb)(rd_kafka_share_t *, + rd_kafka_share_partition_offsets_list_t *, + rd_kafka_resp_err_t, + void *)) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char buf[32]; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "share.acknowledgement.mode", ack_mode); + + rd_snprintf(buf, sizeof(buf), "%d", socket_timeout_ms); + test_conf_set(conf, "socket.timeout.ms", buf); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + if (cb && cb_state) { + rd_kafka_error_t *error = + rd_kafka_share_set_acknowledgement_commit_cb(rkshare, cb, + cb_state); + TEST_ASSERT(error == NULL, + "Failed to set acknowledgement commit callback: " + "%s", + rd_kafka_error_string(error)); + } + return rkshare; +} + +/** + * @brief Consume up to \p expected records (across one or more + * share_poll calls), acknowledging the first \p ack_first. + * + * Caller owns the returned batch via \p *out_batch and must + * rd_kafka_messages_destroy() it. The flattened \p rkmessages array + * is populated with pointers borrowed from the batch (lifetime tied + * to the batch). Asserts that exactly \p expected records were + * received within the polling budget. Used both for the pre-stage + * Phase 1 consume (broker has all records ready, typically arrives + * in a single batch) and for the post-teardown Phase 2 consume + * (records may trickle in across multiple batches as the new + * session warms up). + */ +static void consume_first_batch(rd_kafka_share_t *rkshare, + rd_kafka_message_t **rkmessages, + int expected, + rd_kafka_messages_t **out_batch) { + size_t rcvd = 0; + int attempts = 0; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + int j; + + /* Spin until the first non-empty batch arrives. The test + * pre-stages the broker with all \p expected records so the + * batch we receive will contain exactly that many. */ + while (rcvd == 0 && attempts++ < 30) { + rd_kafka_messages_destroy(batch); + batch = NULL; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + TEST_SAY("consume_first_batch: err=%s\n", + rd_kafka_err2name(rd_kafka_error_code(error))); + rd_kafka_error_destroy(error); + } + rcvd = rd_kafka_messages_count(batch); + } + + TEST_ASSERT((int)rcvd == expected, + "Expected %d records in first batch, got %zu", expected, + rcvd); + + for (j = 0; j < (int)rcvd; j++) { + rkmessages[j] = rd_kafka_messages_get(batch, j); + TEST_ASSERT(!rkmessages[j]->err, + "Unexpected per-record err: %s", + rd_kafka_err2str(rkmessages[j]->err)); + } + + *out_batch = batch; +} + +/** + * @brief Drain pending ack callbacks via commit_async until at least + * \p min_callbacks have been observed, or \p timeout_ms elapses. + * + * commit_async's first step drains rk_rep for callbacks. With no + * pending acks to commit, the remainder is a no-op. + * + * @returns rd_true if min_callbacks observed within timeout. + */ +static rd_bool_t drain_callbacks_via_commit_async(rd_kafka_share_t *rkshare, + test_ack_cb_state_t *cb_state, + int min_callbacks, + int timeout_ms) { + int elapsed_ms = 0; + + while (elapsed_ms < timeout_ms) { + if (cb_state->callback_cnt >= min_callbacks) + return rd_true; + rd_kafka_error_t *err = rd_kafka_share_commit_async(rkshare); + if (err) + rd_kafka_error_destroy(err); + rd_usleep(50 * 1000, NULL); + elapsed_ms += 50; + } + return cb_state->callback_cnt >= min_callbacks; +} + +/** + * @brief Test commit_sync under a wire-level socket timeout, then + * consume + commit a fresh batch. + * + * The three timer layers (api timeout A, RTT R, socket.timeout.ms S) + * determine which event wins: + * - A smallest: commit_sync timeout cb stamps REQUEST_TIMED_OUT + * on results and returns at t=A. Wire request continues in + * flight until either R completes or S fires; late callback + * reflects the actual wire outcome (NO_ERROR if R < S, + * __TIMED_OUT if S < R). + * - S smallest: wire torn down at t=S. Reply handler stamps + * __TIMED_OUT on batches and results; commit_sync returns at + * ~t=S with __TIMED_OUT and callbacks fire with __TIMED_OUT. + * - R smallest: broker reply arrives normally; commit_sync + * returns NO_ERROR. Not exercised by this matrix. + * + * Phase 1 measures the wait between commit_sync return and all ack + * callbacks landing. That wait equals max(0, min(R, S) - + * min(A, R, S)) and is asserted with tolerance. + * + * Phase 2 produces N new records, consumes + acks + commits on the + * (possibly reconnected) consumer. Expects NO_ERROR for every + * partition and a fresh set of NO_ERROR callbacks. + * + * TODO KIP-932: add multi-broker variant once single-broker matrix + * is stable. Multi-broker exercises the case where only one broker + * is slow. + * + * TODO KIP-932: add partial-ack variant — Phase 1 acks only half + * the records, Phase 2 acks the remaining. With session drop on + * connection tear-down, the remaining records reference a dropped + * session and Phase 2's commit_sync surfaces INVALID_RECORD_STATE. + */ +static void do_test_socket_timeout_full_ack_then_more(int api_timeout_ms, + int socket_timeout_ms, + int rtt_ms) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + char topic[128]; + char group[128]; + const int partitions_total = SOCKET_TIMEOUT_MATRIX_PARTITIONS; + const int msgcnt = + partitions_total * SOCKET_TIMEOUT_MATRIX_MSGS_PER_PARTITION; + rd_kafka_message_t **rkmessages; + rd_kafka_messages_t *phase1_batch = NULL; + rd_kafka_messages_t *phase2_batch = NULL; + test_ack_cb_state_t cb_state = {0}; + rd_ts_t t_p1_end_us, t_callbacks_done_us; + int actual_wait_ms, expected_wait_ms; + int min_between_rtt_ms_socket_timeout_ms; + int min_between_api_timeout_ms_rtt_ms_socket_timeout_ms; + rd_kafka_resp_err_t expected_phase1_commit_err; + rd_kafka_resp_err_t expected_phase1_callback_err; + int prev_callback_cnt; + int i; + /* Valgrind serializes threads and slows wall-clock-bound paths; + * widen the tolerance to avoid flaky over-budget failures. */ + const int wait_tolerance_ms = + !strcmp(test_mode, "valgrind") ? 2000 : 500; + + SUB_TEST_QUICK("api_timeout_ms=%d socket_timeout_ms=%d rtt_ms=%d", + api_timeout_ms, socket_timeout_ms, rtt_ms); + + /* Derive expected outcomes from the ordering of three timers: + * + * api_timeout_ms -> commit_sync API's deadline parameter; + * upper bound on how long commit_sync + * can block before returning + * REQUEST_TIMED_OUT. + * rtt_ms -> mock-injected broker round-trip-time; + * time after which the broker's + * ShareAcknowledge response is allowed + * to leave the broker side (records are + * applied broker-side immediately on + * request receive — see + * rdkafka_mock_handlers.c:3811-3840). + * socket_timeout_ms -> wire-level socket.timeout.ms; after + * this the connection is torn down and + * any in-flight request fails with + * __TIMED_OUT (default socket.max.fails + * of 1 makes a single failure terminal). + * + * The expected wait between commit_sync returning and the last + * ack callback landing equals + * max(0, min_between(rtt_ms, socket_timeout_ms) + * - min_between(api_timeout_ms, rtt_ms, + * socket_timeout_ms)) + * + * The reasoning: + * + * - commit_sync returns at + * t_p1_end = min_between(api_timeout_ms, rtt_ms, + * socket_timeout_ms): + * api_timeout_ms smallest: api timer cb fires, stamps + * REQUEST_TIMED_OUT on the per-partition results; + * send_response wakes commit_sync at ~t=api_timeout_ms. + * socket_timeout_ms smallest: wire socket.timeout.ms + * fires, broker thread reply path runs with + * rko_err=__TIMED_OUT, callbacks dispatched, results + * stamped via apply_result, commit_sync returns at + * ~t=socket_timeout_ms. + * rtt_ms smallest: broker reply arrives normally, + * callbacks dispatched with broker's per-partition err, + * results stamped, commit_sync returns at ~t=rtt_ms + * (NO_ERROR for this happy path — not in our matrix). + * + * - All Phase 1 callbacks have landed by + * t_callbacks_done = min_between(rtt_ms, socket_timeout_ms): + * When the wire request resolves the broker-thread reply + * handler dispatches the per-partition callbacks. That + * happens when broker actually replies (t=rtt_ms) OR + * when socket.timeout.ms fires (t=socket_timeout_ms), + * whichever comes first. For + * api_timeout_ms < (rtt_ms, socket_timeout_ms) the + * callbacks fire AFTER commit_sync returned; for + * (rtt_ms or socket_timeout_ms) < api_timeout_ms the + * callbacks already fired BEFORE commit_sync returned + * and the wait is 0. + * + * - Wait = t_callbacks_done - t_p1_end + * = min_between(rtt_ms, socket_timeout_ms) + * - min_between(api_timeout_ms, rtt_ms, + * socket_timeout_ms). + * + * The expected callback err depends on which layer resolved + * the wire: + * - rtt_ms < socket_timeout_ms: broker actually replied — + * callback gets broker's per-partition result (NO_ERROR, + * the records were applied). + * - socket_timeout_ms < rtt_ms: wire torn down before broker + * could reply — helper stamps __TIMED_OUT on batches; the + * app-facing funnel translates it to REQUEST_TIMED_OUT + * before the callback fires. + * + * The expected commit_sync result err comes from which layer + * stamped results first: + * - api_timeout_ms < min_between(rtt_ms, socket_timeout_ms): + * api timer cb wrote REQUEST_TIMED_OUT into results. + * - socket_timeout_ms <= api_timeout_ms: broker-thread reply + * path wrote __TIMED_OUT into results via apply_result; + * the funnel translates it to REQUEST_TIMED_OUT. */ + min_between_rtt_ms_socket_timeout_ms = + socket_timeout_ms < rtt_ms ? socket_timeout_ms : rtt_ms; + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms = api_timeout_ms; + if (rtt_ms < min_between_api_timeout_ms_rtt_ms_socket_timeout_ms) + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms = rtt_ms; + if (socket_timeout_ms < + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms) + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms = + socket_timeout_ms; + expected_wait_ms = min_between_rtt_ms_socket_timeout_ms - + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms; + + if (rtt_ms < api_timeout_ms && rtt_ms < socket_timeout_ms) { + /* rtt_ms smaller than both other timers: broker reply + * lands before either timer fires. commit_sync sees + * the broker's per-partition success and callbacks + * fire with NO_ERROR. Happy path — included for + * matrix completeness, not exercising any timeout + * layer. */ + expected_phase1_commit_err = RD_KAFKA_RESP_ERR_NO_ERROR; + expected_phase1_callback_err = RD_KAFKA_RESP_ERR_NO_ERROR; + } else if (socket_timeout_ms < api_timeout_ms) { + /* socket_timeout_ms < api_timeout_ms (and rtt_ms is + * NOT strictly smallest by the first branch): socket + * fires before api can. (At api == socket the race + * outcome is non-deterministic across runs; that + * boundary case is not in the matrix.) + * + * __TIMED_OUT from the broker-thread socket timer is + * translated to REQUEST_TIMED_OUT at the app-facing + * funnel. */ + expected_phase1_commit_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + expected_phase1_callback_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + } else if (rtt_ms < socket_timeout_ms) { + /* api wins (api <= socket AND rtt < socket; api + * fires no later than broker reply at rtt). Late + * broker reply at rtt brings actual per-partition + * outcome (NO_ERROR — broker did apply the ack) to + * the callback. */ + expected_phase1_commit_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + expected_phase1_callback_err = RD_KAFKA_RESP_ERR_NO_ERROR; + } else { + /* api wins (api <= socket AND rtt >= socket). Socket + * fires before broker can reply, wire torn down; the + * late wire-failure err is __TIMED_OUT, translated to + * REQUEST_TIMED_OUT at the app-facing funnel. */ + expected_phase1_commit_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + expected_phase1_callback_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + } + + ctx = test_ctx_new(); + rd_kafka_mock_sharegroup_set_record_lock_duration( + ctx.mcluster, SOCKET_TIMEOUT_MATRIX_RECORD_LOCK_MS); + + rd_snprintf(topic, sizeof(topic), "0182-fullack-a%d-s%d", + api_timeout_ms, socket_timeout_ms); + rd_snprintf(group, sizeof(group), "sg-0182-fullack-a%d-s%d", + api_timeout_ms, socket_timeout_ms); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, + partitions_total, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + for (i = 0; i < partitions_total; i++) + mock_produce_partition( + ctx.producer, topic, i, + SOCKET_TIMEOUT_MATRIX_MSGS_PER_PARTITION); + + rkshare = create_share_consumer_socket_timeout( + ctx.bootstraps, group, "explicit", socket_timeout_ms, &cb_state, + test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + rkmessages = rd_calloc(msgcnt, sizeof(*rkmessages)); + + /* Phase 1: consume the first batch (all msgcnt records + * given the small partition count and broker readiness), + * acknowledge everyone, then inject RTT and commit_sync. */ + consume_first_batch(rkshare, rkmessages, msgcnt, &phase1_batch); + for (i = 0; i < msgcnt; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + rd_kafka_mock_broker_set_rtt(ctx.mcluster, -1, rtt_ms); + + partitions = NULL; + error = + rd_kafka_share_commit_sync(rkshare, api_timeout_ms, &partitions); + t_p1_end_us = test_clock(); + + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 1: expected non-NULL partition results"); + TEST_ASSERT(partitions->cnt == partitions_total, + "Phase 1: expected %d partition results, got %d", + partitions_total, partitions->cnt); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("Phase 1 commit_sync: %s [%" PRId32 "]: %s\n", + rktpar->topic, rktpar->partition, + rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == expected_phase1_commit_err, + "Phase 1: expected %s, got %s", + rd_kafka_err2name(expected_phase1_commit_err), + rd_kafka_err2name(rktpar->err)); + } + rd_kafka_topic_partition_list_destroy(partitions); + + /* Drain ack callbacks via commit_async until we observe one + * per partition. For socket_timeout_ms-smallest orderings the + * callbacks fired before commit_sync returned so this returns + * immediately. For api_timeout_ms-smallest orderings the + * callbacks fire when the wire resolves (broker reply at + * t=rtt_ms OR socket timer at t=socket_timeout_ms). */ + TEST_ASSERT(drain_callbacks_via_commit_async(rkshare, &cb_state, + partitions_total, 30000), + "Phase 1: expected %d ack callbacks within 30s, got %d", + partitions_total, cb_state.callback_cnt); + t_callbacks_done_us = test_clock(); + + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == + expected_phase1_callback_err, + "Phase 1: expected callback err %s, got %s", + rd_kafka_err2name(expected_phase1_callback_err), + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + + actual_wait_ms = (int)((t_callbacks_done_us - t_p1_end_us) / 1000); + TEST_SAY( + "Phase 1 wait t_callbacks_done - t_p1_end = %dms " + "(expected ~%dms, tolerance %dms)\n", + actual_wait_ms, expected_wait_ms, wait_tolerance_ms); + TEST_ASSERT(actual_wait_ms >= expected_wait_ms - wait_tolerance_ms && + actual_wait_ms <= expected_wait_ms + wait_tolerance_ms, + "Phase 1 wait %dms outside expected %dms +/- %dms", + actual_wait_ms, expected_wait_ms, wait_tolerance_ms); + + prev_callback_cnt = cb_state.callback_cnt; + + /* Phase 2: clear RTT (so Phase 2's broker response isn't + * delayed), produce more, consume + ack + commit. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, -1, 0); + + for (i = 0; i < partitions_total; i++) + mock_produce_partition( + ctx.producer, topic, i, + SOCKET_TIMEOUT_MATRIX_MSGS_PER_PARTITION); + + /* Free phase-1 messages. */ + rd_kafka_messages_destroy(phase1_batch); + phase1_batch = NULL; + memset(rkmessages, 0, msgcnt * sizeof(*rkmessages)); + + consume_first_batch(rkshare, rkmessages, msgcnt, &phase2_batch); + for (i = 0; i < msgcnt; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 5000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 2: expected non-NULL partition results"); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("Phase 2: %s [%" PRId32 "]: %s\n", rktpar->topic, + rktpar->partition, rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == RD_KAFKA_RESP_ERR_NO_ERROR, + "Phase 2: expected NO_ERROR on a fresh " + "session, got %s", + rd_kafka_err2name(rktpar->err)); + } + TEST_ASSERT(partitions->cnt == partitions_total, + "Phase 2: expected %d partition results, got %d", + partitions_total, partitions->cnt); + rd_kafka_topic_partition_list_destroy(partitions); + + TEST_ASSERT(cb_state.callback_cnt > prev_callback_cnt, + "Phase 2: expected new ack callbacks; before=%d " + "after=%d", + prev_callback_cnt, cb_state.callback_cnt); + + rd_kafka_messages_destroy(phase2_batch); + rd_free(rkmessages); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + test_ack_cb_state_destroy(&cb_state); + + SUB_TEST_PASS(); +} + +/** + * @brief Same Phase 1 as do_test_socket_timeout_full_ack_then_more, but + * Phase 1 acks only half the consumed records. Phase 2 then + * acks the remaining half (still in the client's local inflight + * map) and runs commit_sync on those. + * + * After the Phase 1 drain-callbacks step we clear RTT, so Phase 2's + * commit_sync gets a prompt broker response (no second wire-level + * timer to race). + * + * Phase 2 outcome depends on whether the connection was torn down + * during Phase 1, which is equivalent to socket_timeout_ms < rtt_ms: + * + * Connection alive (rtt_ms < socket_timeout_ms): broker session + * preserved; the remaining 15 records are still ACQUIRED for + * this member on the same session. Phase 2's ShareAck advances + * the session epoch normally; broker returns NO_ERROR per + * partition. + * + * Connection killed (socket_timeout_ms < rtt_ms): broker dropped + * the session on TCP close. The remaining 15 records' ACQUIRED + * state was released to AVAILABLE by `release_member_locks`. + * Phase 2's broker-thread reconnects and sends ShareAck with + * session epoch reset to 0 client-side. Broker rejects with + * INVALID_SHARE_SESSION_EPOCH at top-level (no active session + * for this member at epoch 0 outside of a ShareFetch). The + * top-level err propagates to all partitions via the existing + * `rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err` + * helper. + * + * TODO KIP-932: when SHARE_SESSION_NOT_FOUND / + * INVALID_SHARE_SESSION_EPOCH are translated to + * INVALID_RECORD_STATE at the app-facing boundary, + * expected_phase2_commit_err for the killed branch becomes + * INVALID_RECORD_STATE. + */ +static void +do_test_socket_timeout_partial_ack_then_remaining(int api_timeout_ms, + int socket_timeout_ms, + int rtt_ms) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + char topic[128]; + char group[128]; + const int partitions_total = SOCKET_TIMEOUT_MATRIX_PARTITIONS; + const int msgcnt = + partitions_total * SOCKET_TIMEOUT_MATRIX_MSGS_PER_PARTITION; + rd_kafka_message_t **rkmessages; + rd_kafka_messages_t *phase1_batch = NULL; + test_ack_cb_state_t cb_state = {0}; + rd_ts_t t_p1_end_us, t_callbacks_done_us; + int actual_wait_ms, expected_wait_ms; + int min_between_rtt_ms_socket_timeout_ms; + int min_between_api_timeout_ms_rtt_ms_socket_timeout_ms; + rd_kafka_resp_err_t expected_phase1_commit_err; + rd_kafka_resp_err_t expected_phase1_callback_err; + rd_kafka_resp_err_t expected_phase2_commit_err; + rd_bool_t connection_killed; + int prev_callback_cnt; + int phase1_partition_cnt; + int i; + /* See do_test_socket_timeout_full_ack_then_more for the Valgrind + * tolerance rationale. */ + const int wait_tolerance_ms = + !strcmp(test_mode, "valgrind") ? 2000 : 500; + + SUB_TEST_QUICK("api_timeout_ms=%d socket_timeout_ms=%d rtt_ms=%d", + api_timeout_ms, socket_timeout_ms, rtt_ms); + + /* Phase 1 expected outcomes — same derivation as + * do_test_socket_timeout_full_ack_then_more. See the comment + * there for the full reasoning. */ + min_between_rtt_ms_socket_timeout_ms = + socket_timeout_ms < rtt_ms ? socket_timeout_ms : rtt_ms; + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms = api_timeout_ms; + if (rtt_ms < min_between_api_timeout_ms_rtt_ms_socket_timeout_ms) + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms = rtt_ms; + if (socket_timeout_ms < + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms) + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms = + socket_timeout_ms; + expected_wait_ms = min_between_rtt_ms_socket_timeout_ms - + min_between_api_timeout_ms_rtt_ms_socket_timeout_ms; + + if (rtt_ms < api_timeout_ms && rtt_ms < socket_timeout_ms) { + expected_phase1_commit_err = RD_KAFKA_RESP_ERR_NO_ERROR; + expected_phase1_callback_err = RD_KAFKA_RESP_ERR_NO_ERROR; + } else if (socket_timeout_ms < api_timeout_ms) { + /* __TIMED_OUT from the broker-thread socket timer is + * translated to REQUEST_TIMED_OUT at the app-facing + * funnel. */ + expected_phase1_commit_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + expected_phase1_callback_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + } else if (rtt_ms < socket_timeout_ms) { + expected_phase1_commit_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + expected_phase1_callback_err = RD_KAFKA_RESP_ERR_NO_ERROR; + } else { + /* Late wire-failure callback err __TIMED_OUT is + * translated to REQUEST_TIMED_OUT at the app-facing + * funnel. */ + expected_phase1_commit_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + expected_phase1_callback_err = + RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT; + } + + /* Connection torn down iff socket fires before broker reply. + * + * TODO KIP-932: when SHARE_SESSION_NOT_FOUND / + * INVALID_SHARE_SESSION_EPOCH are translated to + * INVALID_RECORD_STATE at the app-facing boundary, the + * killed-branch expected err becomes INVALID_RECORD_STATE. */ + connection_killed = socket_timeout_ms < rtt_ms; + expected_phase2_commit_err = + connection_killed ? RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH + : RD_KAFKA_RESP_ERR_NO_ERROR; + + ctx = test_ctx_new(); + rd_kafka_mock_sharegroup_set_record_lock_duration( + ctx.mcluster, SOCKET_TIMEOUT_MATRIX_RECORD_LOCK_MS); + + rd_snprintf(topic, sizeof(topic), "0182-partial-a%d-s%d-r%d", + api_timeout_ms, socket_timeout_ms, rtt_ms); + rd_snprintf(group, sizeof(group), "sg-0182-partial-a%d-s%d-r%d", + api_timeout_ms, socket_timeout_ms, rtt_ms); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, + partitions_total, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + for (i = 0; i < partitions_total; i++) + mock_produce_partition( + ctx.producer, topic, i, + SOCKET_TIMEOUT_MATRIX_MSGS_PER_PARTITION); + + rkshare = create_share_consumer_socket_timeout( + ctx.bootstraps, group, "explicit", socket_timeout_ms, &cb_state, + test_share_ack_cb); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + rkmessages = rd_calloc(msgcnt, sizeof(*rkmessages)); + + /* Phase 1: consume the first batch (all msgcnt records), + * then ack the first half of the batch in receive order. + * The exact set of partitions covered depends on broker-side + * record ordering; the assertions below adapt to whatever + * partitions appear in commit_sync's results. */ + consume_first_batch(rkshare, rkmessages, msgcnt, &phase1_batch); + for (i = 0; i < msgcnt / 2; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + rd_kafka_mock_broker_set_rtt(ctx.mcluster, -1, rtt_ms); + + partitions = NULL; + error = + rd_kafka_share_commit_sync(rkshare, api_timeout_ms, &partitions); + t_p1_end_us = test_clock(); + + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 1: expected non-NULL partition results"); + + /* The number of partitions reflects whichever ones had records + * in the first half of the batch — depends on broker-side + * record ordering. We assert each partition's err matches the + * expected code, and drain that many callbacks. */ + phase1_partition_cnt = partitions->cnt; + TEST_ASSERT(phase1_partition_cnt > 0, + "Phase 1: expected at least one partition result"); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("Phase 1 commit_sync: %s [%" PRId32 "]: %s\n", + rktpar->topic, rktpar->partition, + rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == expected_phase1_commit_err, + "Phase 1: expected %s, got %s", + rd_kafka_err2name(expected_phase1_commit_err), + rd_kafka_err2name(rktpar->err)); + } + rd_kafka_topic_partition_list_destroy(partitions); + + TEST_ASSERT(drain_callbacks_via_commit_async( + rkshare, &cb_state, phase1_partition_cnt, 30000), + "Phase 1: expected %d ack callbacks within 30s, got %d", + phase1_partition_cnt, cb_state.callback_cnt); + t_callbacks_done_us = test_clock(); + + TEST_ASSERT(test_ack_cb_state_first_err(&cb_state) == + expected_phase1_callback_err, + "Phase 1: expected callback err %s, got %s", + rd_kafka_err2name(expected_phase1_callback_err), + rd_kafka_err2name(test_ack_cb_state_first_err(&cb_state))); + + actual_wait_ms = (int)((t_callbacks_done_us - t_p1_end_us) / 1000); + TEST_SAY( + "Phase 1 wait t_callbacks_done - t_p1_end = %dms " + "(expected ~%dms, tolerance %dms)\n", + actual_wait_ms, expected_wait_ms, wait_tolerance_ms); + TEST_ASSERT(actual_wait_ms >= expected_wait_ms - wait_tolerance_ms && + actual_wait_ms <= expected_wait_ms + wait_tolerance_ms, + "Phase 1 wait %dms outside expected %dms +/- %dms", + actual_wait_ms, expected_wait_ms, wait_tolerance_ms); + + prev_callback_cnt = cb_state.callback_cnt; + + /* Phase 2: clear RTT and ack the remaining records (still in + * the local inflight map from Phase 1's consume — the second + * half per partition). No new consume_batch in this variant. + * + * Small settle delay so the consumer's broker thread can + * finish post-teardown bookkeeping (reconnect, session + * reset, broker decommission cleanup) before Phase 2's + * commit_sync probes the state. Without this perm 5/6/boundary + * cases racing the post-teardown handling can return + * __STATE from the FANOUT op instead of the expected + * INVALID_SHARE_SESSION_EPOCH from the local epoch-0 check. */ + rd_kafka_mock_broker_set_rtt(ctx.mcluster, -1, 0); + rd_sleep(1); + + /* Ack the remaining half of the batch in receive order. */ + for (i = msgcnt / 2; i < msgcnt; i++) + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 5000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, + "Phase 2: expected non-NULL partition results"); + + /* As in Phase 1, the partition count reflects whichever + * partitions had records in the second half. Some partitions + * may appear in both phases (e.g., partition that had its + * first 5 records acked in Phase 1 and last 5 in Phase 2). */ + TEST_ASSERT(partitions->cnt > 0, + "Phase 2: expected at least one partition result"); + + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + TEST_SAY("Phase 2 commit_sync: %s [%" PRId32 "]: %s\n", + rktpar->topic, rktpar->partition, + rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == expected_phase2_commit_err, + "Phase 2: expected %s, got %s", + rd_kafka_err2name(expected_phase2_commit_err), + rd_kafka_err2name(rktpar->err)); + } + TEST_ASSERT( + drain_callbacks_via_commit_async( + rkshare, &cb_state, prev_callback_cnt + partitions->cnt, 10000), + "Phase 2: expected %d new ack callbacks; before=%d " + "after=%d", + partitions->cnt, prev_callback_cnt, cb_state.callback_cnt); + rd_kafka_topic_partition_list_destroy(partitions); + + TEST_ASSERT( + cb_state.errs[cb_state.callback_cnt - 1] == + expected_phase2_commit_err, + "Phase 2: expected last callback err %s, got %s", + rd_kafka_err2name(expected_phase2_commit_err), + rd_kafka_err2name(cb_state.errs[cb_state.callback_cnt - 1])); + + rd_kafka_messages_destroy(phase1_batch); + rd_free(rkmessages); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + test_ack_cb_state_destroy(&cb_state); + + SUB_TEST_PASS(); +} + +/* =================================================================== + * Topic-level metadata error tests. + * + * Verify that a share consumer surfaces topic-level errors from + * metadata responses to the application as rd_kafka_error_t via + * consume_batch. Only TOPIC_EXCEPTION and TOPIC_AUTHORIZATION_FAILED + * reach the app; transient codes (UNKNOWN_TOPIC, UNKNOWN_TOPIC_OR_PART, + * UNKNOWN_TOPIC_ID, UNKNOWN_PARTITION) are not delivered. Repeats of + * the same (topic, err) are deduped; recovery, unsubscribe, and + * re-subscribe each have well-defined behaviour exercised below. + * + * These scenarios are hard to reproduce on a real broker — the mock + * cluster lets us inject the exact per-topic error byte in a metadata + * response on demand. + * =================================================================== */ + +/* Drive at least one successful consume_batch so the share assignment + * is fully materialised before the test injects an error. Records are + * ACKed inline because the consumer is in explicit-ack mode. */ +static void share_topic_err_prime_assignment(rd_kafka_share_t *rkshare) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + size_t rcvd; + size_t j; + int attempts; + rd_bool_t got_any = rd_false; + + for (attempts = 0; attempts < 20; attempts++) { + error = rd_kafka_share_poll(rkshare, 1000, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + rd_kafka_share_acknowledge(rkshare, rkm); + got_any = rd_true; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + if (got_any) + return; + } + TEST_FAIL( + "Pre-condition: expected to consume a batch before " + "injecting the metadata error"); +} + +/* Force a metadata refresh on the share consumer's underlying rk so + * the injected per-topic err is observed. */ +static void share_topic_err_force_metadata(rd_kafka_share_t *rkshare) { + const rd_kafka_metadata_t *md = NULL; + rd_kafka_t *rk; + + rk = test_share_consumer_get_rk(rkshare); + (void)rd_kafka_metadata(rk, 1 /*all_topics*/, NULL, &md, 5000); + if (md) + rd_kafka_metadata_destroy(md); +} + +/* Drain consume_batch until either the expected err code surfaces (then + * return rd_true) or `max_attempts` calls go by without it (return + * rd_false). Records that arrive are destroyed. */ +static rd_bool_t share_topic_err_wait_for_err(rd_kafka_share_t *rkshare, + rd_kafka_resp_err_t expected, + int max_attempts) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + size_t rcvd; + size_t j; + int attempts; + + for (attempts = 0; attempts < max_attempts; attempts++) { + error = rd_kafka_share_poll(rkshare, 500, &batch); + if (error) { + rd_kafka_resp_err_t code = rd_kafka_error_code(error); + TEST_SAY("share_poll returned %s: %s\n", + rd_kafka_err2name(code), + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + if (code == expected) + return rd_true; + continue; + } + rcvd = rd_kafka_messages_count(batch); + /* Ack received records so the next share_poll can + * proceed past the explicit-mode "previous poll + * unacked" gate. */ + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + rd_kafka_share_acknowledge(rkshare, rkm); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + return rd_false; +} + +/* Run `n_attempts` consume_batch calls and fail the test if any of them + * returns an rd_kafka_error_t — used for the negative-assertion tests + * (transient-code log-only paths must not surface). */ +static void share_topic_err_assert_no_err(rd_kafka_share_t *rkshare, + int n_attempts, + const char *context) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + size_t rcvd; + size_t j; + int attempts; + + for (attempts = 0; attempts < n_attempts; attempts++) { + error = rd_kafka_share_poll(rkshare, 200, &batch); + if (error) { + rd_kafka_resp_err_t code = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + TEST_FAIL( + "[%s] unexpected error from share_poll: " + "%s", + context, rd_kafka_err2name(code)); + } + rcvd = rd_kafka_messages_count(batch); + /* Ack received records so the next share_poll can + * proceed past the explicit-mode "previous poll + * unacked" gate. */ + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + rd_kafka_share_acknowledge(rkshare, rkm); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } +} + +/* Run one share-topic-err scenario: assign, inject `inject_err`, + * verify consume_batch surfaces `expect_err` (or fails). */ +static void do_test_share_topic_err_surfaces(const char *topic_suffix, + rd_kafka_resp_err_t inject_err, + rd_kafka_resp_err_t expect_err) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + char topic[64]; + char group[64]; + + SUB_TEST_QUICK("inject=%s expect=%s", rd_kafka_err2name(inject_err), + rd_kafka_err2name(expect_err)); + + ctx = test_ctx_new(); + rd_snprintf(topic, sizeof(topic), "0182-%s", topic_suffix); + rd_snprintf(group, sizeof(group), "sg-0182-%s", topic_suffix); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + mock_produce(ctx.producer, topic, 5); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, inject_err); + share_topic_err_force_metadata(rkshare); + + TEST_ASSERT(share_topic_err_wait_for_err(rkshare, expect_err, 30), + "Expected consume_batch to surface %s within 30 attempts", + rd_kafka_err2name(expect_err)); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +static void test_share_consumer_surfaces_topic_exception(void) { + do_test_share_topic_err_surfaces("surfaces-topic-exception", + RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION, + RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION); +} + + +static void test_share_consumer_surfaces_topic_authorization_failed(void) { + do_test_share_topic_err_surfaces( + "surfaces-topic-auth-failed", + RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED, + RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); +} + + +/* A topic with N partitions failing in a single metadata cycle must + * surface exactly one op for that topic, not one per partition. + * + * share_toppar_enq_error keys its accumulator on topic_id and + * dedups-on-add, so the per-rktp calls from partition_cnt_update and + * propagate_notexists (2N total) collapse to a single entry in + * rkcg_errored_topics — and hence a single rd_kafka_consumer_err. */ +static void test_share_consumer_multi_partition_single_op_per_cycle(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + const char *topic = "0182-multipart-single-op"; + const char *group = "sg-0182-multipart-single-op"; + const int partition_cnt = 5; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + size_t rcvd, j; + int err_count = 0; + int attempts; + + SUB_TEST(); + + ctx = test_ctx_new(); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, + partition_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic with %d partitions", partition_cnt); + mock_produce(ctx.producer, topic, partition_cnt * 3); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + /* Inject AUTH_FAILED on the topic; partition_cnt_update + + * propagate_notexists will each fire enq_error per rktp. */ + rd_kafka_mock_topic_set_error( + ctx.mcluster, topic, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); + share_topic_err_force_metadata(rkshare); + + /* Drain a short window immediately after the synchronous + * force_metadata. By the time it returns, propagate has emitted + * the (single) op for this cycle. The window is intentionally + * short to keep it within one heartbeat interval and count just + * what this cycle produced. */ + for (attempts = 0; attempts < 3; attempts++) { + error = rd_kafka_share_poll(rkshare, 200, &batch); + if (error) { + rd_kafka_resp_err_t code = rd_kafka_error_code(error); + const char *errstr = rd_kafka_error_string(error); + if (code == + RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED && + errstr && strstr(errstr, topic)) + err_count++; + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) + rd_kafka_share_acknowledge(rkshare, rkm); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(err_count == 1, + "expected exactly 1 AUTH_FAILED op for a %d-partition " + "topic in a single metadata cycle, got %d", + partition_cnt, err_count); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* The same topic failing with a different error code must surface a + * fresh op for the new code rather than being deduped against the + * previous one. */ +static void test_share_consumer_re_emits_when_err_code_changes(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + const char *topic = "0182-err-code-change"; + const char *group = "sg-0182-err-code-change"; + + SUB_TEST(); + + ctx = test_ctx_new(); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + mock_produce(ctx.producer, topic, 5); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + /* First err: AUTH_FAILED. */ + rd_kafka_mock_topic_set_error( + ctx.mcluster, topic, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT( + share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED, 30), + "First AUTH_FAILED must surface"); + + /* Second err for the same topic: TOPIC_EXCEPTION (different + * code) — must surface, dedup must NOT swallow it. */ + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT(share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION, 30), + "TOPIC_EXCEPTION must surface after AUTH_FAILED " + "(err code change must bypass dedup)"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* A topic that surfaces AUTH_FAILED, then recovers, then fails again + * with the same code must surface the error a second time. */ +static void test_share_consumer_re_surfaces_after_recovery(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + const char *topic = "0182-re-surface-after-recovery"; + const char *group = "sg-0182-re-surface-after-recovery"; + + SUB_TEST(); + + ctx = test_ctx_new(); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + mock_produce(ctx.producer, topic, 5); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + /* Phase 1: fail — first surface. */ + rd_kafka_mock_topic_set_error( + ctx.mcluster, topic, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT( + share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED, 30), + "First AUTH_FAILED must surface"); + + /* Phase 2: recover — no error must reach the app. */ + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_NO_ERROR); + share_topic_err_force_metadata(rkshare); + share_topic_err_assert_no_err(rkshare, 5, + "no error must surface while recovered"); + + /* Phase 3: fail again with the same code — must surface a + * second time. */ + rd_kafka_mock_topic_set_error( + ctx.mcluster, topic, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT( + share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED, 30), + "AUTH_FAILED must surface a second time after recovery"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* A topic that surfaces TOPIC_EXCEPTION, then recovers, then fails + * again with the same code must surface the error a second time. */ +static void +test_share_consumer_re_surfaces_after_recovery_topic_exception(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + const char *topic = "0182-re-surface-topic-exception"; + const char *group = "sg-0182-re-surface-topic-exception"; + + SUB_TEST(); + + ctx = test_ctx_new(); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + mock_produce(ctx.producer, topic, 5); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + /* Phase 1: fail. */ + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT(share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION, 30), + "first TOPIC_EXCEPTION must surface"); + + /* Phase 2: recover. */ + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_NO_ERROR); + share_topic_err_force_metadata(rkshare); + share_topic_err_assert_no_err( + rkshare, 5, "no error must surface while topic is recovered"); + + /* Phase 3: re-fail with the same code — must surface again. */ + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT(share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION, 30), + "TOPIC_EXCEPTION must surface a second time after " + "recovery"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* A topic surfaces TOPIC_EXCEPTION, gets unsubscribed (no surface), + * and is then re-subscribed while still failing. The error must + * surface again on re-subscribe — not be permanently suppressed. */ +static void test_share_consumer_resubscribe_re_emits_persistent_failure(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + const char *topic = "0182-resubscribe-re-emit"; + const char *group = "sg-0182-resubscribe-re-emit"; + + SUB_TEST(); + + ctx = test_ctx_new(); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + mock_produce(ctx.producer, topic, 5); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + /* Phase 1: subscribe + fail + surface. */ + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION); + share_topic_err_force_metadata(rkshare); + TEST_ASSERT(share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION, 30), + "first TOPIC_EXCEPTION must surface"); + + /* Phase 2: unsubscribe. */ + TEST_ASSERT(rd_kafka_share_unsubscribe(rkshare) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "unsubscribe"); + + /* Phase 3: re-subscribe to the same still-failing topic; the + * error must surface again. Re-force metadata across the wait + * loop so the request happens after the share-assignment + * heartbeat re-populates the partition list. */ + test_share_consumer_subscribe_multi(rkshare, 1, topic); + rd_bool_t saw_err = rd_false; + int outer; + for (outer = 0; outer < 10 && !saw_err; outer++) { + share_topic_err_force_metadata(rkshare); + if (share_topic_err_wait_for_err( + rkshare, RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION, 3)) + saw_err = rd_true; + } + TEST_ASSERT(saw_err, + "TOPIC_EXCEPTION must surface again after " + "re-subscribing to a still-failing topic"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* UNKNOWN_TOPIC_OR_PART is debug-logged only and must not reach the + * app via consume_batch. */ +static void test_share_consumer_does_not_surface_unknown_topic_or_part(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + const char *topic = "0182-no-surface-unknown-tp"; + const char *group = "sg-0182-no-surface-unknown-tp"; + + SUB_TEST(); + + ctx = test_ctx_new(); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + mock_produce(ctx.producer, topic, 5); + + /* Custom consumer with the metadata propagation defer window + * disabled so the no-surface path is exercised on the first + * metadata refresh rather than 30 s later. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "topic.metadata.propagation.max.ms", "0"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + share_topic_err_prime_assignment(rkshare); + + rd_kafka_mock_topic_set_error(ctx.mcluster, topic, + RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART); + share_topic_err_force_metadata(rkshare); + + share_topic_err_assert_no_err( + rkshare, 20, + "UNKNOWN_TOPIC_OR_PART must be logged only, not surfaced"); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Log callback shared by the two select_broker STATE_UP guard tests + * below. Counts the broker-thread short-circuit log emitted at + * src/rdkafka_broker.c when a SHARE_FETCH op is served against a + * broker whose rkb_state is not STATE_UP. + * =================================================================== */ +static void no_bounce_loop_log_cb(const rd_kafka_t *rk, + int level, + const char *fac, + const char *buf) { + rd_atomic32_t *cnt = rd_kafka_opaque(rk); + if (cnt && !strcmp(fac, "SHAREFETCH") && strstr(buf, "broker not up")) + rd_atomic32_add(cnt, 1); +} + +/* =================================================================== + * do_test_no_bounce_loop_on_down_broker + * + * Steady-state DOWN: the consumer is idle when the broker is taken + * down, sleeps long enough for the broker thread to settle rkb_state + * to !UP, then drives consume_batch for ~1s. Every FANOUT must skip + * the DOWN leader via the select_broker STATE_UP guard, so no + * "broker not up" log line should fire. + * + * Wire never sees a ShareFetch on the DOWN broker (broker thread + * rejects the internal op before any RPC is built), so we assert on + * the count of the broker-thread short-circuit debug log instead of + * on mock_get_requests. + * =================================================================== */ +static void do_test_no_bounce_loop_on_down_broker(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_atomic32_t broker_not_up_cnt; + rd_kafka_error_t *error; + rd_kafka_messages_t *batch = NULL; + const char *topic = "0182-no_bounce_loop"; + const char *group = "sg-0182-no-bounce-loop"; + const int msgcnt_phase1 = 5; + const int msgcnt_phase2 = 5; + int acked, cnt; + size_t rcvd; + size_t share_fetch_cnt_before_drain; + size_t share_fetch_cnt_after_drain; + rd_ts_t end_ts; + + SUB_TEST_QUICK(); + + /* Taking the only broker down legitimately raises + * __ALL_BROKERS_DOWN and __TRANSPORT on producer + consumer + * error callbacks. None of these should fail the test. */ + test_curr->is_fatal_cb = test_error_is_not_fatal_cb; + + ctx = test_ctx_new(); + rd_kafka_mock_start_request_tracking(ctx.mcluster); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce(ctx.producer, topic, msgcnt_phase1); + + rd_atomic32_init(&broker_not_up_cnt, 0); + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "debug", "broker"); + /* Cap reconnect backoff so the consumer recovers quickly after + * set_up. The default max (10s) extends past the recovery + * sleep below, causing the post-recovery ShareFetch not to + * land in time. */ + test_conf_set(conf, "reconnect.backoff.ms", "100"); + test_conf_set(conf, "reconnect.backoff.max.ms", "500"); + rd_kafka_conf_set_log_cb(conf, no_bounce_loop_log_cb); + rd_kafka_conf_set_opaque(conf, &broker_not_up_cnt); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + acked = consume_and_ack_all(rkshare, msgcnt_phase1); + TEST_ASSERT(acked == msgcnt_phase1, "phase1: expected %d acked, got %d", + msgcnt_phase1, acked); + + /* Flush any acks still cached in rkb_share_async_ack_details + * before taking the broker down. Otherwise the next FANOUT + * after set_down would dispatch an ack-only op to the DOWN + * broker (the FANOUT iteration must always deliver cached + * acks so the broker thread can surface the error), and the + * broker thread would legitimately log "broker not up" once. + * That log is unrelated to the select_broker guard. */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + + /* Phase-2 records sit in the partition log; the post-recovery + * consume below drains them. */ + mock_produce(ctx.producer, topic, msgcnt_phase2); + + TEST_SAY("Taking broker 1 down\n"); + rd_kafka_mock_broker_set_down(ctx.mcluster, 1); + + /* Settle: let the client's broker thread detect TCP close and + * transition rkb_state to !UP before we start counting. After + * this point every select_broker reads DOWN — no race window. */ + rd_sleep(1); + rd_atomic32_set(&broker_not_up_cnt, 0); + + end_ts = test_clock() + 1000 * 1000; + while (test_clock() < end_ts) { + error = rd_kafka_share_poll(rkshare, 100, &batch); + TEST_ASSERT(!error, + "unexpected error from share_poll while " + "broker is down: %s", + error ? rd_kafka_error_string(error) : "NULL"); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "expected 0 records while broker is down, " + "got %zu", + rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + cnt = rd_atomic32_get(&broker_not_up_cnt); + TEST_SAY("\"broker not up\" log count: %d (expected 0)\n", cnt); + TEST_ASSERT(cnt == 0, + "select_broker should skip the DOWN leader on every " + "FANOUT once rkb_state has settled; got %d " + "\"broker not up\" log lines", + cnt); + + TEST_SAY("Bringing broker 1 back up\n"); + rd_kafka_mock_broker_set_up(ctx.mcluster, 1); + + /* The main-thread retrigger keeps calling select_broker. As + * soon as broker 1 reaches STATE_UP the next select_broker + * returns it and a ShareFetch fires automatically — records + * land on the consumer queue without any consume_batch call + * driving the fetch. */ + rd_sleep(3); + + share_fetch_cnt_before_drain = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_fetch_request, NULL); + TEST_SAY("ShareFetch count after recovery (pre-drain): %" PRIusz "\n", + share_fetch_cnt_before_drain); + TEST_ASSERT(share_fetch_cnt_before_drain >= 1, + "expected >= 1 ShareFetch via internal retry after " + "set_up, got %" PRIusz, + share_fetch_cnt_before_drain); + + /* Drain the pre-fetched records. They're already on the + * consumer queue, so share_poll returns them directly + * without enqueueing a FANOUT and no new ShareFetch fires. */ + error = rd_kafka_share_poll(rkshare, 100, &batch); + rcvd = rd_kafka_messages_count(batch); + share_fetch_cnt_after_drain = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_fetch_request, NULL); + /* Destroy the batch before asserting so a failed assert can't + * leak it (TEST_FAIL longjmps past any later destroy). */ + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_ASSERT(!error, "post-recovery share_poll error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + TEST_ASSERT(rcvd == (size_t)msgcnt_phase2, + "expected %d records from queue, got %" PRIusz, + msgcnt_phase2, rcvd); + TEST_ASSERT(share_fetch_cnt_after_drain == share_fetch_cnt_before_drain, + "share_poll should drain the queue without firing " + "a new ShareFetch; pre=%" PRIusz " post=%" PRIusz, + share_fetch_cnt_before_drain, share_fetch_cnt_after_drain); + + rd_kafka_mock_clear_requests(ctx.mcluster); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + +/* =================================================================== + * do_test_one_log_on_broker_down_during_active_empty_poll + * + * Race-window companion to do_test_no_bounce_loop_on_down_broker. + * + * With an empty topic the consumer's FANOUT->ShareFetch->Step 6 + * empty-poll loop fires continuously. We rapidly flip the broker + * up/down across N cycles while that loop is hot. Each set_down + * has a narrow window between the mock TCP close and the client + * broker thread updating rkb_state, in which select_broker can + * read stale STATE_UP and enqueue one more op — the broker thread + * (rkb_state has caught up by then) logs "broker not up" at most + * once and replies ERR__STATE. The next Step 6 re-select reads + * DOWN and skips, closing the loop until the next set_down. + * + * Total log count is therefore bounded by N (one slip per + * set_down). Without the select_broker guard, each set_down would + * produce hundreds of logs per second for the duration of the + * down window — unbounded across cycles. + * =================================================================== */ +static void do_test_one_log_on_broker_down_during_active_empty_poll(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_atomic32_t broker_not_up_cnt; + const char *topic = "0182-race_bounce"; + const char *group = "sg-0182-race-bounce"; + const int msgcnt_recovery = 5; + const int n_cycles = 10; + const int max_allowed_log_cnt = n_cycles; + rd_kafka_messages_t *batch = NULL; + size_t rcvd; + rd_kafka_error_t *error; + int i, cnt, acked; + + SUB_TEST_QUICK(); + + /* Taking the only broker down legitimately raises + * __ALL_BROKERS_DOWN and __TRANSPORT on producer + consumer + * error callbacks. None of these should fail the test. */ + test_curr->is_fatal_cb = test_error_is_not_fatal_cb; + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + rd_atomic32_init(&broker_not_up_cnt, 0); + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "debug", "broker"); + /* Cap reconnect backoff so each set_up reconnects quickly + * across the chaos cycles. */ + test_conf_set(conf, "reconnect.backoff.ms", "100"); + test_conf_set(conf, "reconnect.backoff.max.ms", "500"); + rd_kafka_conf_set_log_cb(conf, no_bounce_loop_log_cb); + rd_kafka_conf_set_opaque(conf, &broker_not_up_cnt); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Prime with one record so the share group has joined and the + * partition is assigned by the time we take the broker down. + * Without this guarantee, select_broker would return NULL + * (no toppars) and the bounce loop wouldn't fire at all, + * making the test pass for the wrong reason. */ + mock_produce(ctx.producer, topic, 1); + acked = consume_and_ack_all(rkshare, 1); + TEST_ASSERT(acked == 1, "prime: expected 1 acked, got %d", acked); + + /* Flush the prime ack still cached in rkb_share_async_ack_details + * before taking the broker down. Otherwise the next FANOUT + * iteration would dispatch an ack-only op to the DOWN broker + * and produce a "broker not up" log unrelated to the + * select_broker race we're measuring. */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + + /* Reset to drop any noise from cgrp/connection bring-up. */ + rd_atomic32_set(&broker_not_up_cnt, 0); + + /* Kickstart the empty-poll loop: one share_poll on the now + * empty topic starts the FANOUT->Step 6 cycle which keeps + * firing on the main thread until share_poll returns. */ + error = rd_kafka_share_poll(rkshare, 500, &batch); + TEST_ASSERT(!error, "kickstart share_poll error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, "expected 0 records, got %zu", rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Chaos: rapidly flip the broker up/down across n_cycles + * while the consumer's empty-poll loop is hot. Each set_down + * may admit one race-window slip; total log count must stay + * bounded by n_cycles. */ + for (i = 0; i < n_cycles; i++) { + TEST_SAY("Cycle %d/%d: taking broker 1 down\n", i + 1, + n_cycles); + rd_kafka_mock_broker_set_down(ctx.mcluster, 1); + error = rd_kafka_share_poll(rkshare, 200, &batch); + TEST_ASSERT(!error, "cycle %d down: share_poll error: %s", + i + 1, + error ? rd_kafka_error_string(error) : "NULL"); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "cycle %d down: expected 0 records, got %zu", i + 1, + rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + + TEST_SAY("Cycle %d/%d: bringing broker 1 back up\n", i + 1, + n_cycles); + rd_kafka_mock_broker_set_up(ctx.mcluster, 1); + error = rd_kafka_share_poll(rkshare, 200, &batch); + TEST_ASSERT(!error, "cycle %d up: share_poll error: %s", i + 1, + error ? rd_kafka_error_string(error) : "NULL"); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "cycle %d up: expected 0 records, got %zu", i + 1, + rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + cnt = rd_atomic32_get(&broker_not_up_cnt); + TEST_SAY("\"broker not up\" log count after %d cycles: %d (max %d)\n", + n_cycles, cnt, max_allowed_log_cnt); + TEST_ASSERT(cnt <= max_allowed_log_cnt, + "race-window slips must be bounded to %d (one per " + "set_down across %d cycles); got %d log lines", + max_allowed_log_cnt, n_cycles, cnt); + + /* Recovery: broker is left UP at the end of the chaos loop. + * Produce records and verify the consumer drains them. */ + mock_produce(ctx.producer, topic, msgcnt_recovery); + + acked = consume_and_ack_all(rkshare, msgcnt_recovery); + TEST_ASSERT(acked == msgcnt_recovery, + "post-recovery: expected %d acked, got %d", msgcnt_recovery, + acked); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Log callback for do_test_fetch_stall_logged_on_no_broker. Counts + * the main-thread "Fetch stalled: no eligible broker" debug log. + * =================================================================== */ +static void fetch_stall_log_cb(const rd_kafka_t *rk, + int level, + const char *fac, + const char *buf) { + rd_atomic32_t *cnt = rd_kafka_opaque(rk); + if (cnt && !strcmp(fac, "FETCHMORE") && + strstr(buf, "Fetch stalled: no eligible broker")) + rd_atomic32_add(cnt, 1); +} + +/* =================================================================== + * do_test_fetch_stall_logged_on_no_broker + * + * With a partition assigned but the only broker down, the main-thread + * re-trigger loop wants to fetch (share_fetch_more_records set, no op + * in-flight) yet select_broker returns NULL. Verify it emits the + * rate-limited "Fetch stalled: no eligible broker" log: at least once + * (the stall is reported) and bounded (the throttle prevents a flood + * while the empty-poll loop is hot). + * =================================================================== */ +static void do_test_fetch_stall_logged_on_no_broker(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_atomic32_t stall_cnt; + const char *topic = "0182-fetch-stall"; + const char *group = "sg-0182-fetch-stall"; + const int msgcnt_recovery = 5; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + size_t rcvd; + int acked, cnt; + + SUB_TEST_QUICK(); + + /* Taking the only broker down raises __ALL_BROKERS_DOWN / + * __TRANSPORT on the callbacks; none should fail the test. */ + test_curr->is_fatal_cb = test_error_is_not_fatal_cb; + + ctx = test_ctx_new(); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + rd_atomic32_init(&stall_cnt, 0); + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + /* The stall log is emitted on the CONSUMER debug context. */ + test_conf_set(conf, "debug", "consumer"); + test_conf_set(conf, "reconnect.backoff.ms", "100"); + test_conf_set(conf, "reconnect.backoff.max.ms", "500"); + rd_kafka_conf_set_log_cb(conf, fetch_stall_log_cb); + rd_kafka_conf_set_opaque(conf, &stall_cnt); + + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Prime so the share group joins and the partition is assigned — + * otherwise the stall reason would be "no partitions assigned". */ + mock_produce(ctx.producer, topic, 1); + acked = consume_and_ack_all(rkshare, 1); + TEST_ASSERT(acked == 1, "prime: expected 1 acked, got %d", acked); + + /* Flush the primed ack so the down broker doesn't get an + * ack-only op unrelated to the stall we're measuring. */ + error = rd_kafka_share_commit_async(rkshare); + TEST_ASSERT(!error, "commit_async error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + + /* Kickstart a poll while the broker is still up so the primed ack + * is flushed to the broker — otherwise it is redelivered after + * recovery and inflates the post-recovery count. */ + error = rd_kafka_share_poll(rkshare, 500, &batch); + TEST_ASSERT(!error, "kickstart share_poll error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + rd_kafka_messages_destroy(batch); + batch = NULL; + + /* Reset to drop any bring-up noise. */ + rd_atomic32_set(&stall_cnt, 0); + + /* Take the only broker down, then poll. With the partition still + * assigned and no broker reachable, the main-thread loop keeps + * wanting to fetch but select_broker returns NULL. The 2s poll + * stays inside the throttle window. */ + rd_kafka_mock_broker_set_down(ctx.mcluster, 1); + + error = rd_kafka_share_poll(rkshare, 2000, &batch); + TEST_ASSERT(!error, "down share_poll error: %s", + error ? rd_kafka_error_string(error) : "NULL"); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, "expected 0 records, got %zu", rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + + cnt = rd_atomic32_get(&stall_cnt); + TEST_SAY("\"Fetch stalled: no eligible broker\" log count: %d\n", cnt); + TEST_ASSERT(cnt >= 1, + "expected the stall to be logged at least once, got %d", + cnt); + /* Throttled: a 2s hot loop must not flood. Allow a small margin + * for a race-window reset + re-log around set_down. */ + TEST_ASSERT(cnt <= 3, + "stall log must be throttled (<=3 in a 2s window), got %d", + cnt); + + /* Recovery: bring the broker back and confirm the consumer + * drains. */ + rd_kafka_mock_broker_set_up(ctx.mcluster, 1); + mock_produce(ctx.producer, topic, msgcnt_recovery); + acked = consume_and_ack_all(rkshare, msgcnt_recovery); + TEST_ASSERT(acked == msgcnt_recovery, + "post-recovery: expected %d acked, got %d", msgcnt_recovery, + acked); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + test_curr->is_fatal_cb = NULL; + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Partition-level error injection: API basics. + * =================================================================== */ +static void test_partition_error_injection_general(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + char topic[64]; + char group[64]; + const int msgcnt = 3; + int acked; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + rd_snprintf(topic, sizeof(topic), "0182-part_err_general"); + rd_snprintf(group, sizeof(group), "sg-0182-part_err_general"); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + /* Unknown topics are auto-created. */ + TEST_ASSERT(rd_kafka_mock_partition_push_request_errors( + ctx.mcluster, "0182-no-such-topic", 0, + RD_KAFKAP_ShareFetch, 1, + RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "push to unknown topic"); + + /* Out-of-range partition is rejected. */ + TEST_ASSERT(rd_kafka_mock_partition_push_request_errors( + ctx.mcluster, topic, 99, RD_KAFKAP_ShareFetch, 1, + RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR) == + RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART, + "push to unknown partition"); + + TEST_ASSERT(rd_kafka_mock_partition_push_request_errors( + ctx.mcluster, topic, 0, RD_KAFKAP_ShareFetch, 1, + RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "push partition error"); + + mock_produce_partition(ctx.producer, topic, 0, msgcnt); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* The injected error is transient: all records are delivered + * once it has been consumed off the stack. */ + acked = consume_and_ack_all(rkshare, msgcnt); + TEST_ASSERT(acked == msgcnt, "expected %d acked, got %d", msgcnt, + acked); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * ShareFetch partition error injection: errors on one partition + * must not affect the other, and the partition recovers once the + * error stack drains. + * =================================================================== */ +static void test_partition_error_injection_share_fetch(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + char topic[64]; + char group[64]; + const int msgs_per_part = 5; + int acked; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + rd_snprintf(topic, sizeof(topic), "0182-part_err_sharefetch"); + rd_snprintf(group, sizeof(group), "sg-0182-part_err_sharefetch"); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 2, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce_partition(ctx.producer, topic, 0, msgs_per_part); + mock_produce_partition(ctx.producer, topic, 1, msgs_per_part); + + TEST_ASSERT(rd_kafka_mock_partition_push_request_errors( + ctx.mcluster, topic, 1, RD_KAFKAP_ShareFetch, 2, + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER, + RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "push partition errors"); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Partition 0 is unaffected and partition 1 recovers after + * the two errored fetches: nothing is lost. */ + acked = consume_and_ack_all(rkshare, 2 * msgs_per_part); + TEST_ASSERT(acked == 2 * msgs_per_part, "expected %d acked, got %d", + 2 * msgs_per_part, acked); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * ShareAcknowledge partition error injection: the injected error + * surfaces in commit_sync results for that partition only. + * =================================================================== */ +static void test_partition_error_injection_share_ack(void) { + test_ctx_t ctx; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *partitions = NULL; + rd_kafka_error_t *error; + char topic[64]; + char group[64]; + const int msgs_per_part = 5; + const rd_kafka_resp_err_t injected_err = + RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR; + int acked; + int i; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(); + + rd_snprintf(topic, sizeof(topic), "0182-part_err_shareack"); + rd_snprintf(group, sizeof(group), "sg-0182-part_err_shareack"); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 2, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + mock_produce_partition(ctx.producer, topic, 0, msgs_per_part); + mock_produce_partition(ctx.producer, topic, 1, msgs_per_part); + + rkshare = create_mock_share_consumer(ctx.bootstraps, group, "explicit", + NULL, NULL); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + acked = consume_and_ack_all(rkshare, 2 * msgs_per_part); + TEST_ASSERT(acked == 2 * msgs_per_part, "expected %d acked, got %d", + 2 * msgs_per_part, acked); + + TEST_ASSERT(rd_kafka_mock_partition_push_request_errors( + ctx.mcluster, topic, 0, RD_KAFKAP_ShareAcknowledge, 1, + injected_err) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push partition error"); + + partitions = NULL; + error = rd_kafka_share_commit_sync(rkshare, 30000, &partitions); + if (error) + rd_kafka_error_destroy(error); + + TEST_ASSERT(partitions != NULL, "expected non-NULL partition results"); + TEST_ASSERT(partitions->cnt == 2, + "expected results for 2 partitions, got %d", + partitions->cnt); + + /* Partition 0 carries the injected error, partition 1 is + * unaffected. */ + for (i = 0; i < partitions->cnt; i++) { + rd_kafka_topic_partition_t *rktpar = &partitions->elems[i]; + rd_kafka_resp_err_t exp_err = rktpar->partition == 0 + ? injected_err + : RD_KAFKA_RESP_ERR_NO_ERROR; + + TEST_SAY("%s [%" PRId32 "]: %s\n", rktpar->topic, + rktpar->partition, rd_kafka_err2name(rktpar->err)); + TEST_ASSERT(rktpar->err == exp_err, + "partition [%" PRId32 "]: expected %s, got %s", + rktpar->partition, rd_kafka_err2name(exp_err), + rd_kafka_err2name(rktpar->err)); + } + + rd_kafka_topic_partition_list_destroy(partitions); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +int main_0182_share_consumer_error_handling_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + test_timeout_set(120); + + test_commit_sync_share_session_not_found(); + test_commit_sync_invalid_share_session_epoch(); + test_commit_sync_share_session_limit_reached(); + test_commit_sync_group_authorization_failed(); + test_commit_sync_topic_authorization_failed(); + test_commit_sync_invalid_request(); + test_commit_sync_multi_partition_top_level_error(); + test_consume_batch_multi_partition_top_level_error(); + test_commit_sync_at_epoch_zero_returns_invalid_session_epoch_error(); + test_consume_batch_at_epoch_zero_strips_piggyback_acks(); + test_strip_pre_set_survives_sharefetch_err(); + + /* Topic-level metadata err surface */ + test_share_consumer_surfaces_topic_exception(); + test_share_consumer_surfaces_topic_authorization_failed(); + test_share_consumer_multi_partition_single_op_per_cycle(); + test_share_consumer_re_emits_when_err_code_changes(); + test_share_consumer_re_surfaces_after_recovery(); + test_share_consumer_re_surfaces_after_recovery_topic_exception(); + test_share_consumer_resubscribe_re_emits_persistent_failure(); + test_share_consumer_does_not_surface_unknown_topic_or_part(); + + /* Socket timeout matrix (single broker). + * + * Each call corresponds to a different ordering of the three + * timer layers (api_timeout_ms, socket_timeout_ms, rtt_ms). + * See the comment in do_test_socket_timeout_full_ack_then_more + * for the full analysis of what each ordering exercises. + * Arguments are (api_timeout_ms, socket_timeout_ms, rtt_ms). */ + + /* All 6 strict-inequality permutations. */ + do_test_socket_timeout_full_ack_then_more( + 1000, 5000, 3000); /* api < rtt < socket */ + do_test_socket_timeout_full_ack_then_more( + 1000, 3000, 5000); /* api < socket < rtt */ + do_test_socket_timeout_full_ack_then_more( + 3000, 5000, 1000); /* rtt < api < socket */ + do_test_socket_timeout_full_ack_then_more( + 5000, 3000, 1000); /* rtt < socket < api */ + do_test_socket_timeout_full_ack_then_more( + 3000, 1000, 5000); /* socket < api < rtt */ + do_test_socket_timeout_full_ack_then_more( + 5000, 1000, 3000); /* socket < rtt < api */ + /* Boundary api == socket is intentionally skipped: the race + * between the api timer cb and the wire socket timer is + * non-deterministic in practice — both outcomes are + * observed across runs. */ + + /* Partial-ack variant of the matrix. Phase 1 acks only half + * the consumed records; Phase 2 acks the remaining half (still + * in client's inflight map). Phase 2 commit_sync surfaces + * INVALID_SHARE_SESSION_EPOCH when the session was dropped + * by the Phase 1 socket teardown (socket < rtt), NO_ERROR + * otherwise. */ + do_test_socket_timeout_partial_ack_then_remaining( + 1000, 5000, 3000); /* api < rtt < socket */ + do_test_socket_timeout_partial_ack_then_remaining( + 1000, 3000, 5000); /* api < socket < rtt */ + do_test_socket_timeout_partial_ack_then_remaining( + 3000, 5000, 1000); /* rtt < api < socket */ + do_test_socket_timeout_partial_ack_then_remaining( + 5000, 3000, 1000); /* rtt < socket < api */ + do_test_socket_timeout_partial_ack_then_remaining( + 3000, 1000, 5000); /* socket < api < rtt */ + do_test_socket_timeout_partial_ack_then_remaining( + 5000, 1000, 3000); /* socket < rtt < api */ + /* Boundary api == socket skipped — see comment above. */ + + /* select_broker STATE_UP guard: steady-state DOWN and + * race-window flavours. */ + do_test_no_bounce_loop_on_down_broker(); + do_test_one_log_on_broker_down_during_active_empty_poll(); + do_test_fetch_stall_logged_on_no_broker(); + + + /* Partition-level error injection. */ + test_partition_error_injection_general(); + test_partition_error_injection_share_fetch(); + test_partition_error_injection_share_ack(); + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0183-share_consumer_leader_change_mock.c b/lib/librdkafka-2.15.0/tests/0183-share_consumer_leader_change_mock.c new file mode 100644 index 00000000000..a63ac5f474c --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0183-share_consumer_leader_change_mock.c @@ -0,0 +1,1234 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +/** + * @brief Share consumer leader change tests. + * + * Verifies that per-partition leader-change errors on ShareAcknowledge + * trigger metadata refresh and that unnecessary RPCs are avoided once + * the client discovers the new leader. + */ + +#define CONSUME_ARRAY 1024 + +/* =================================================================== + * Mock broker infrastructure. + * =================================================================== */ +typedef struct test_ctx_s { + rd_kafka_t *producer; + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; +} test_ctx_t; + +static test_ctx_t test_ctx_new(int nbrok) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + char errstr[512]; + + memset(&ctx, 0, sizeof(ctx)); + + ctx.mcluster = test_mock_cluster_new(nbrok, &ctx.bootstraps); + + TEST_ASSERT(rd_kafka_mock_set_apiversion( + ctx.mcluster, RD_KAFKAP_ShareGroupHeartbeat, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareGroupHeartbeat"); + TEST_ASSERT(rd_kafka_mock_set_apiversion(ctx.mcluster, + RD_KAFKAP_ShareFetch, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to enable ShareFetch"); + + rd_kafka_mock_sharegroup_set_auto_offset_reset(ctx.mcluster, 1); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb); + + ctx.producer = + rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(ctx.producer != NULL, "Failed to create producer: %s", + errstr); + + return ctx; +} + +static void test_ctx_destroy(test_ctx_t *ctx) { + if (ctx->producer) + rd_kafka_destroy(ctx->producer); + if (ctx->mcluster) + test_mock_cluster_destroy(ctx->mcluster); + memset(ctx, 0, sizeof(*ctx)); +} + +static void subscribe_topics(rd_kafka_share_t *rkshare, + const char **topics, + int topic_cnt) { + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + int i; + + subs = rd_kafka_topic_partition_list_new(topic_cnt); + for (i = 0; i < topic_cnt; i++) + rd_kafka_topic_partition_list_add(subs, topics[i], + RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(rkshare, subs); + TEST_ASSERT(!err, "subscribe failed: %s", rd_kafka_err2str(err)); + rd_kafka_topic_partition_list_destroy(subs); +} + +static void subscribe_one(rd_kafka_share_t *rkshare, const char *topic) { + subscribe_topics(rkshare, &topic, 1); +} + +static int mock_produce(rd_kafka_t *producer, + const char *topic, + int32_t partition, + int msgcnt) { + int i; + for (i = 0; i < msgcnt; i++) { + char payload[64]; + snprintf(payload, sizeof(payload), "%s-p%" PRId32 "-%d", topic, + partition, i); + TEST_ASSERT(rd_kafka_producev( + producer, RD_KAFKA_V_TOPIC(topic), + RD_KAFKA_V_PARTITION(partition), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), + RD_KAFKA_V_END) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Produce to %s [%" PRId32 "] failed", topic, + partition); + } + rd_kafka_flush(producer, 5000); + return msgcnt; +} + +static rd_bool_t is_share_ack_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == + RD_KAFKAP_ShareAcknowledge; +} + +static rd_bool_t is_share_fetch_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == RD_KAFKAP_ShareFetch; +} + +static rd_bool_t is_metadata_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == RD_KAFKAP_Metadata; +} + + +/* =================================================================== + * Test — ShareAcknowledge per-partition NOT_LEADER_OR_FOLLOWER + * should trigger metadata refresh and avoid redundant RPCs. + * + * Setup: + * - 2 brokers, 1 topic, 1 partition on broker 1 + * - Explicit ack mode, background metadata refresh disabled + * - Produce 10 records, consume one batch (expect all 10) + * - Move leader to broker 2 + * + * Loop (5 iterations): + * - Acknowledge 2 records from the held batch + * - commit_sync → attempts to route the batch to the cached + * leader. Round 1 routes to broker 1 (cache is stale); + * rounds 2-5 are short-circuited locally (see Expected + * behavior below). + * + * Expected behavior with this PR: + * - The first commit_sync sends a ShareAcknowledge to broker 1 + * (cache still says broker 1 is leader). The response carries + * NOT_LEADER_OR_FOLLOWER plus a CurrentLeader hint and + * NodeEndpoints; the inline metadata update applies the new + * leader to the cache without a separate Metadata RPC. + * - Rounds 2-5 are caught by the local leader-stale short-circuit + * in rd_kafka_share_ack_batch_resolve_leader_or_fail_acks and + * fail without sending an RPC. + * - All 5 commit_sync return NOT_LEADER_OR_FOLLOWER. + * - 1 ShareAcknowledge RPC total, 0 Metadata RPCs. + * =================================================================== */ +static void test_shareack_leader_change_reduces_rpcs(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + const char *topic = "0183-shareack-nlof"; + const char *group = "sg-0183-shareack-nlof"; + const int broker1 = 1; + const int broker2 = 2; + const int msgcnt = 10; + const int acks_per_commit = 2; + const int commit_rounds = msgcnt / acks_per_commit; + rd_kafka_messages_t *batch = NULL; + int round; + size_t rcvd = 0; + size_t consumed_idx = 0; + size_t share_ack_cnt; + size_t metadata_cnt; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(2); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, + broker1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set initial leader to broker %d", broker1); + + mock_produce(ctx.producer, topic, RD_KAFKA_PARTITION_UA, msgcnt); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "-1"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + subscribe_one(rkshare, topic); + + /* Consume one batch — expect all 10 records in a single call. */ + error = rd_kafka_share_poll(rkshare, 10000, &batch); + TEST_ASSERT(!error, "share_poll failed: %s", + error ? rd_kafka_error_string(error) : ""); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == (size_t)msgcnt, + "expected %d records in first batch, got %" PRIusz, msgcnt, + rcvd); + + /* Move leader to broker 2 before acknowledging. */ + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, + broker2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader to broker %d", broker2); + + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + /* 5 rounds: acknowledge 2 records, commitSync each time. + * Each commitSync should get NOT_LEADER_OR_FOLLOWER. */ + for (round = 0; round < commit_rounds; round++) { + rd_kafka_topic_partition_list_t *results = NULL; + int k; + + for (k = 0; + k < acks_per_commit && consumed_idx < (size_t)msgcnt; + k++, consumed_idx++) { + rd_kafka_resp_err_t ack_err; + ack_err = rd_kafka_share_acknowledge( + rkshare, + rd_kafka_messages_get(batch, consumed_idx)); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "round %d: acknowledge failed: %s", round, + rd_kafka_err2str(ack_err)); + } + + error = rd_kafka_share_commit_sync(rkshare, 500, &results); + + TEST_SAY("round %d: commit_sync returned %s\n", round, + error ? rd_kafka_error_string(error) : "success"); + + TEST_ASSERT(results != NULL && results->cnt > 0, + "round %d: expected results from commit_sync", + round); + + for (k = 0; k < results->cnt; k++) { + TEST_SAY(" round %d: partition %s [%" PRId32 + "] err=%s\n", + round, results->elems[k].topic, + results->elems[k].partition, + rd_kafka_err2name(results->elems[k].err)); + TEST_ASSERT( + results->elems[k].err == + RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER, + "round %d: expected NOT_LEADER_OR_FOLLOWER, " + "got %s", + round, rd_kafka_err2name(results->elems[k].err)); + } + + rd_kafka_topic_partition_list_destroy(results); + + if (error) + rd_kafka_error_destroy(error); + } + + share_ack_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_ack_request, NULL); + metadata_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_metadata_request, NULL); + + TEST_SAY("ShareAcknowledge requests after leader change: %" PRIusz "\n", + share_ack_cnt); + TEST_SAY("Metadata requests after leader change: %" PRIusz "\n", + metadata_cnt); + + /* The first commit_sync sends a ShareAcknowledge RPC to the + * stale broker; the response carries CurrentLeader and + * NodeEndpoints which update the metadata cache inline. + * Rounds 2-5 are caught by the local leader-stale check and + * fail without sending an RPC. No separate Metadata RPC is + * needed because the cache update is inline. */ + TEST_ASSERT(share_ack_cnt == 1, + "expected 1 ShareAcknowledge RPC, got %" PRIusz, + share_ack_cnt); + TEST_ASSERT(metadata_cnt == 0, + "expected 0 Metadata RPCs (inline cache update), " + "got %" PRIusz, + metadata_cnt); + + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + /* Clean up held message handles. */ + rd_kafka_messages_destroy(batch); + batch = NULL; + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/* =================================================================== + * Test — per-partition NOT_LEADER_OR_FOLLOWER on ShareFetch is + * handled silently: no OP_CONSUMER_ERR, consumer recovers + * and continues reading from the new leader. + * + * Two-broker mock cluster. Phase 1 produces and consumes records + * from broker1. The partition leader is moved to broker2. Phase 2 + * verifies the consumer silently recovers and reads new records + * from broker2. + * + * Wire-level verification: at least 1 Metadata request is triggered + * (error handler fired metadata refresh) and ShareFetch count stays + * below 10 (no retry storm). The NOT_LEADER_OR_FOLLOWER response + * may arrive before or after request tracking starts, so a >= 2 + * ShareFetch lower bound is not reliable; >= 1 is the meaningful + * minimum. + * + * TODO KIP-932: Add a real-broker companion test that triggers a + * leader change on a multi-broker cluster. + * =================================================================== */ +static void test_partition_not_leader_or_follower_silent(void) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + const char *topic = "0183-nlof-silent"; + const char *group = "sg-0183-nlof-silent"; + const int broker1 = 1; + const int broker2 = 2; + const int msgcnt = 5; + rd_kafka_messages_t *batch = NULL; + int phase1_consumed = 0; + int phase2_consumed = 0; + int error_cnt = 0; + int attempts = 0; + size_t rcvd = 0; + size_t share_fetch_cnt; + size_t metadata_cnt; + size_t j; + + SUB_TEST_QUICK(); + + ctx = test_ctx_new(2); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + /* Explicitly set the initial leader (do not assume a default). */ + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, + broker1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set initial leader to broker %d", broker1); + + /* Phase 1: produce and consume records from broker1. + * auto.offset.reset=earliest (set by test_ctx_new) so the + * share group starts from the beginning of the log. */ + mock_produce(ctx.producer, topic, RD_KAFKA_PARTITION_UA, msgcnt); + + /* Disable background metadata refresh so that the only metadata + * refresh that fires is the one triggered by the per-partition + * NOT_LEADER_OR_FOLLOWER error. This makes the >= 2 ShareFetch + * assertion deterministic: background refresh would otherwise + * silently update the cached leader before Phase 2 starts. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "implicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "-1"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + subscribe_one(rkshare, topic); + + while (phase1_consumed < msgcnt && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, j); + if (m && !m->err) + phase1_consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(phase1_consumed == msgcnt, + "Phase 1: expected %d records from broker %d, got %d", + msgcnt, broker1, phase1_consumed); + + /* Start request tracking before the leader change to capture + * ShareFetch requests sent after it. */ + rd_kafka_mock_start_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + /* Move the partition leader to broker2. The consumer's next + * ShareFetch to broker1 will get NOT_LEADER_OR_FOLLOWER per + * partition, which triggers a silent metadata refresh and + * reconnect to broker2. */ + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, topic, 0, + broker2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader to broker %d", broker2); + + /* Produce records that will be available on broker2. mock_produce + * flushes before returning, so all records are committed to + * broker2 before Phase 2 begins. */ + mock_produce(ctx.producer, topic, RD_KAFKA_PARTITION_UA, msgcnt); + + /* Phase 2: consume records after recovery. OP_CONSUMER_ERR + * surfaces as a non-NULL return from consume_batch; silent + * handling must keep that return NULL throughout recovery. */ + attempts = 0; + while (phase2_consumed < msgcnt && attempts++ < 30) { + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + error_cnt++; + rd_kafka_error_destroy(error); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, j); + if (m && !m->err) + phase2_consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(phase2_consumed >= msgcnt, + "Phase 2: expected >= %d records from broker %d after " + "recovery, got %d", + msgcnt, broker2, phase2_consumed); + TEST_ASSERT(error_cnt == 0, + "expected 0 OP_CONSUMER_ERR (NOT_LEADER_OR_FOLLOWER " + "must be silently handled), got %d", + error_cnt); + + /* Verify the error handler fired (metadata refresh triggered) and + * did not cause a retry storm. The NOT_LEADER_OR_FOLLOWER response + * may be processed before or after request tracking starts, so only + * a Metadata lower bound (not a >= 2 ShareFetch) is reliable. */ + share_fetch_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_share_fetch_request, NULL); + metadata_cnt = test_mock_get_matching_request_cnt( + ctx.mcluster, is_metadata_request, NULL); + TEST_SAY("ShareFetch requests after leader change: %" PRIusz "\n", + share_fetch_cnt); + TEST_SAY("Metadata requests after leader change: %" PRIusz "\n", + metadata_cnt); + TEST_ASSERT(metadata_cnt >= 1, + "expected >= 1 Metadata request after leader change " + "(error handler must have triggered metadata refresh), " + "got %" PRIusz, + metadata_cnt); + TEST_ASSERT(share_fetch_cnt >= 1, + "expected >= 1 ShareFetch request after leader change, " + "got %" PRIusz, + share_fetch_cnt); + TEST_ASSERT(share_fetch_cnt < 10, + "expected < 10 ShareFetch requests (no retry storm), " + "got %" PRIusz, + share_fetch_cnt); + + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + + SUB_TEST_PASS(); +} + + +/** + * @brief Drive a consume loop until \p expected records have been read + * or the attempt budget is exhausted. In implicit mode an extra + * zero-timeout consume_batch flushes the last batch's auto-ack + * before returning. + */ +static int consume_phase(rd_kafka_share_t *rkshare, + rd_bool_t explicit_mode, + rd_bool_t use_commit_sync, + int expected, + int post_first_batch_settle_ms, + const char *phase_name) { + int consumed = 0; + int attempts = 0; + int max_attempts = expected * 3 + 60; + rd_kafka_messages_t *batch = NULL; + rd_kafka_topic_partition_list_t *flush_results = NULL; + rd_kafka_error_t *flush_err; + rd_bool_t first_batch = rd_true; + int ri; + + while (consumed < expected && attempts++ < max_attempts) { + size_t rcvd = 0; + rd_kafka_error_t *error; + size_t j; + + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + TEST_SAY("%s: share_poll err %s\n", phase_name, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, j); + if (m && !m->err) { + consumed++; + TEST_SAY("%s: record #%d: %s [%" PRId32 + "] offset %" PRId64 "\n", + phase_name, consumed, + rd_kafka_topic_name(m->rkt), + m->partition, m->offset); + if (explicit_mode) + rd_kafka_share_acknowledge(rkshare, m); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + if (explicit_mode && rcvd > 0) { + if (use_commit_sync) { + rd_kafka_topic_partition_list_t *results = NULL; + rd_kafka_error_t *cerr; + TEST_SAY( + "%s: calling commit_sync " + "(consumed %d)\n", + phase_name, consumed); + cerr = rd_kafka_share_commit_sync(rkshare, 5000, + &results); + if (cerr) { + TEST_SAY("%s: commit_sync error: %s\n", + phase_name, + rd_kafka_error_string(cerr)); + rd_kafka_error_destroy(cerr); + } + RD_IF_FREE( + results, + rd_kafka_topic_partition_list_destroy); + } else { + rd_kafka_error_t *cerr = + rd_kafka_share_commit_async(rkshare); + if (cerr) { + TEST_SAY("%s: commit_async error: %s\n", + phase_name, + rd_kafka_error_string(cerr)); + rd_kafka_error_destroy(cerr); + } + } + } + + /* After the first consume_batch call, let any pending + * metadata refresh and per-broker session + * reconciliation triggered by the leader change settle + * before the next consume_batch picks the next broker + * in round-robin. Without this wait the next broker's + * session is still stale, producing a redundant + * NOT_LEADER_FOR_PARTITION and metadata refresh. */ + if (first_batch && post_first_batch_settle_ms > 0) { + rd_usleep(post_first_batch_settle_ms * 1000, NULL); + first_batch = rd_false; + } + } + + /* Flush before exiting: commit_sync auto-acks for implicit + * mode (via acknowledge_all_if_implicit) and drains pending + * acks for explicit mode. Using commit_sync for both modes + * avoids re-arming share_fetch_more_records via another + * consume_batch, which would keep the main-thread fetch loop + * spinning between phases. */ + TEST_SAY("%s: calling commit_sync flush (consumed %d/%d)\n", phase_name, + consumed, expected); + flush_err = rd_kafka_share_commit_sync(rkshare, 5000, &flush_results); + if (flush_err) { + TEST_SAY("%s: commit_sync flush error: %s\n", phase_name, + rd_kafka_error_string(flush_err)); + rd_kafka_error_destroy(flush_err); + } else { + TEST_SAY("%s: commit_sync flush succeeded\n", phase_name); + } + if (flush_results) { + for (ri = 0; ri < flush_results->cnt; ri++) { + rd_kafka_topic_partition_t *rp = + &flush_results->elems[ri]; + TEST_SAY("%s: commit_sync result: %s [%" PRId32 + "]: %s\n", + phase_name, rp->topic, rp->partition, + rd_kafka_err2str(rp->err)); + } + rd_kafka_topic_partition_list_destroy(flush_results); + flush_results = NULL; + } + + TEST_SAY("%s: consumed %d (expected == %d) in %d attempts\n", + phase_name, consumed, expected, attempts); + return consumed; +} + + +/* =================================================================== + * Test — produce / consume / leader-change / produce / consume on a + * multi-topic, multi-partition mock cluster. Verifies that + * the share consumer picks up the new leader either via the + * Share RPC error path (refresh disabled) or via background + * metadata refresh (refresh enabled). + * + * Setup: + * - 3 brokers, 2 topics ("recovery-a" with 2 partitions, + * "recovery-b" with 5 partitions). Initial leaders round-robin. + * + * Phase 1: + * - Produce a random 10-20 records per partition. + * - Consume them all, ack/commit per variant. + * + * Leader change: + * - Each partition's leader is rotated to a different broker + * (chosen randomly). + * - If \p wait_for_metadata_refresh is true the test sleeps long + * enough for the configured 500 ms background refresh to fire. + * + * Phase 2: + * - Produce a random 10-20 records per partition. + * - Consume them all. + * - Assert the consumed count equals the Phase 2 production. + * + * TODO KIP-932: once the ShareFetch response's per-partition + * CurrentLeader / NodeEndpoints hints are applied inline to the + * metadata cache (same approach as the ShareAcknowledge path), the + * refresh-disabled variant must see 0 Metadata RPCs after the + * leader change and the refresh-enabled variant must see >= 1 + * (background refresh). Today both variants see >= 1 because the + * ShareFetch error path forces a metadata refresh RPC until the + * inline update lands. + * =================================================================== */ +static void +do_test_leader_change_consume_recovery(rd_bool_t explicit_mode, + rd_bool_t use_commit_sync, + rd_bool_t wait_for_metadata_refresh, + const char *variant_name) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char *topic_a; + char *topic_b; + const char *topics[2]; + const int part_a = 2; + const int part_b = 5; + const char *group = "sg-0183-recovery"; + int phase1_total = 0; + int phase2_total = 0; + int phase1_consumed = 0; + int phase2_consumed = 0; + int leader_before_a[2]; + int leader_before_b[5]; + size_t metadata_window1; + size_t metadata_window2; + int i; + int cnt; + + SUB_TEST_QUICK("%s", variant_name); + + topic_a = rd_strdup(test_mk_topic_name("0183-recovery-a", 1)); + topic_b = rd_strdup(test_mk_topic_name("0183-recovery-b", 1)); + topics[0] = topic_a; + topics[1] = topic_b; + + ctx = test_ctx_new(3); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_a, part_a, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic a"); + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic_b, part_b, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic b"); + + for (i = 0; i < part_a; i++) { + leader_before_a[i] = (i % 3) + 1; + TEST_ASSERT(rd_kafka_mock_partition_set_leader( + ctx.mcluster, topic_a, i, leader_before_a[i]) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader a[%d] -> %d", i, leader_before_a[i]); + } + for (i = 0; i < part_b; i++) { + leader_before_b[i] = (i % 3) + 1; + TEST_ASSERT(rd_kafka_mock_partition_set_leader( + ctx.mcluster, topic_b, i, leader_before_b[i]) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader b[%d] -> %d", i, leader_before_b[i]); + } + + srand(42); + + for (i = 0; i < part_a; i++) { + cnt = 10 + (rand() % 11); + phase1_total += mock_produce(ctx.producer, topic_a, i, cnt); + TEST_SAY( + "%s: Phase 1 produce: %s [%d] = %d records " + "(offsets 0..%d)\n", + variant_name, topic_a, i, cnt, cnt - 1); + } + for (i = 0; i < part_b; i++) { + cnt = 10 + (rand() % 11); + phase1_total += mock_produce(ctx.producer, topic_b, i, cnt); + TEST_SAY( + "%s: Phase 1 produce: %s [%d] = %d records " + "(offsets 0..%d)\n", + variant_name, topic_b, i, cnt, cnt - 1); + } + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", + explicit_mode ? "explicit" : "implicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", + wait_for_metadata_refresh ? "10000" : "-1"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "create share consumer"); + subscribe_topics(rkshare, topics, 2); + + phase1_consumed = consume_phase(rkshare, explicit_mode, use_commit_sync, + phase1_total, 0, "Phase 1"); + TEST_ASSERT(phase1_consumed == phase1_total, + "Phase 1: expected %d records, got %d", phase1_total, + phase1_consumed); + + /* Let initial offset queries (ListOffsetsRequest from the + * share consumer's reused fetch-state machine on partition + * assignment) complete before changing leaders. Without this + * settle window, an in-flight ListOffsets response could race + * with the leader change and bring in an extra metadata + * refresh that pollutes Window 1. + * TODO KIP-932: this wait can be removed once the share + * consumer no longer issues ListOffsetsRequests. */ + rd_usleep(2000 * 1000, NULL); + + /* Window 1 — track Metadata RPCs from the leader change + * through the wait sleep. Captures any background metadata + * refresh fired while the consumer was idle. */ + rd_kafka_mock_start_request_tracking(ctx.mcluster); + + for (i = 0; i < part_a; i++) { + int new_leader = leader_before_a[i]; + while (new_leader == leader_before_a[i]) + new_leader = 1 + (rand() % 3); + TEST_SAY("%s: leader change: %s [%d]: B%d -> B%d\n", + variant_name, topic_a, i, leader_before_a[i], + new_leader); + TEST_ASSERT(rd_kafka_mock_partition_set_leader( + ctx.mcluster, topic_a, i, new_leader) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader a[%d] -> %d", i, new_leader); + } + for (i = 0; i < part_b; i++) { + int new_leader = leader_before_b[i]; + while (new_leader == leader_before_b[i]) + new_leader = 1 + (rand() % 3); + TEST_SAY("%s: leader change: %s [%d]: B%d -> B%d\n", + variant_name, topic_b, i, leader_before_b[i], + new_leader); + TEST_ASSERT(rd_kafka_mock_partition_set_leader( + ctx.mcluster, topic_b, i, new_leader) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader b[%d] -> %d", i, new_leader); + } + + if (wait_for_metadata_refresh) + rd_usleep(11000 * 1000, NULL); + + metadata_window1 = test_mock_get_matching_request_cnt( + ctx.mcluster, is_metadata_request, NULL); + TEST_SAY( + "%s: Window 1 (post leader change, pre Phase 2) " + "Metadata: %" PRIusz "\n", + variant_name, metadata_window1); + + /* Pause tracking around Phase 2 produce so producer-driven + * metadata refreshes do not pollute the Window 2 count. */ + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + for (i = 0; i < part_a; i++) { + cnt = 10 + (rand() % 11); + phase2_total += mock_produce(ctx.producer, topic_a, i, cnt); + TEST_SAY( + "%s: Phase 2 produce: %s [%d] = %d records " + "(Phase 2 starts at offset after Phase 1)\n", + variant_name, topic_a, i, cnt); + } + for (i = 0; i < part_b; i++) { + cnt = 10 + (rand() % 11); + phase2_total += mock_produce(ctx.producer, topic_b, i, cnt); + TEST_SAY( + "%s: Phase 2 produce: %s [%d] = %d records " + "(Phase 2 starts at offset after Phase 1)\n", + variant_name, topic_b, i, cnt); + } + + /* Ensure all produces and their triggered metadata refreshes + * have settled before measuring Phase 2 consume. */ + rd_kafka_flush(ctx.producer, 5000); + + /* Window 2 — track Metadata RPCs strictly around Phase 2 + * consume. Excludes producer activity from Phase 2 produce. */ + rd_kafka_mock_start_request_tracking(ctx.mcluster); + + phase2_consumed = consume_phase(rkshare, explicit_mode, use_commit_sync, + phase2_total, 1000, "Phase 2"); + TEST_ASSERT(phase2_consumed == phase2_total, + "Phase 2: expected %d records after leader change, " + "got %d", + phase2_total, phase2_consumed); + + metadata_window2 = test_mock_get_matching_request_cnt( + ctx.mcluster, is_metadata_request, NULL); + TEST_SAY("%s: Window 2 (Phase 2 consume) Metadata: %" PRIusz "\n", + variant_name, metadata_window2); + + rd_kafka_mock_stop_request_tracking(ctx.mcluster); + rd_kafka_mock_clear_requests(ctx.mcluster); + + /* TODO KIP-932: the share consumer still runs the regular + * consumer's offset-query path. On leader change, the failed + * ListOffsets requests trigger extra metadata refreshes that + * inflate the wait-refresh window1 count and the no-refresh + * window2 count beyond the single refresh those windows are + * meant to capture. Once the offset-query path is removed + * from the share consumer, tighten the >= 1 checks below to + * == 1. + * + * TODO KIP-932: revisit these assertions when the ShareFetch + * response handler applies its per-partition CurrentLeader + * and NodeEndpoints hints inline to the metadata cache (same + * approach already taken for the ShareAcknowledge path). At + * that point the no-refresh variant must see 0 Metadata RPCs + * during Phase 2 consume — the cache is updated from the + * Share response, no separate Metadata RPC is needed — and + * the wait-refresh expectations stay as today. */ + if (wait_for_metadata_refresh) { + TEST_ASSERT(metadata_window1 >= 1, + "%s: expected >= 1 Metadata RPC during the " + "wait window (background refresh), got %" PRIusz, + variant_name, metadata_window1); + TEST_ASSERT(metadata_window2 == 0, + "%s: expected 0 Metadata RPCs during Phase 2 " + "consume (cache already updated, no error path), " + "got %" PRIusz, + variant_name, metadata_window2); + } else { + TEST_ASSERT(metadata_window1 == 0, + "%s: expected 0 Metadata RPCs before Phase 2 " + "(background refresh disabled, no error yet), " + "got %" PRIusz, + variant_name, metadata_window1); + TEST_ASSERT(metadata_window2 >= 1, + "%s: expected >= 1 Metadata RPC during Phase 2 " + "consume (ShareFetch error path triggered " + "refresh), got %" PRIusz, + variant_name, metadata_window2); + } + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + rd_free(topic_a); + rd_free(topic_b); + + SUB_TEST_PASS(); +} + + +// /* =================================================================== +// * Test — per-partition UNKNOWN_TOPIC_OR_PARTITION on ShareFetch is +// * handled silently: no OP_CONSUMER_ERR is delivered after the +// * topic is deleted. +// * +// * Single-broker mock cluster. Phase 1 produces and consumes records +// * to establish a share session. The topic is deleted. Phase 2 polls +// * several times and verifies no error is surfaced to the application. +// * +// * Wire-level verification: ShareFetch count is between 1 and 9 +// * (lower bound: error path fired; upper bound: no retry storm). +// * +// * TODO KIP-932: Add a real-broker companion test that deletes a +// * live topic and verifies the same silent behaviour. +// * =================================================================== */ +// static void test_partition_unknown_topic_silent(void) { +// test_ctx_t ctx; +// rd_kafka_conf_t *conf; +// rd_kafka_share_t *rkshare; +// rd_kafka_error_t *error; +// const char *topic = "0183-utp-silent"; +// const char *group = "sg-0183-utp-silent"; +// const int msgcnt = 5; +// rd_kafka_message_t *rkmessages[CONSUME_ARRAY]; +// int phase1_consumed = 0; +// int error_cnt = 0; +// int attempts = 0; +// int poll_cnt = 0; +// size_t rcvd = 0; +// size_t share_fetch_cnt; +// size_t metadata_cnt; +// size_t j; + +// SUB_TEST_QUICK(); + +// ctx = test_ctx_new(1); + +// TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, 1, 1) == +// RD_KAFKA_RESP_ERR_NO_ERROR, +// "create topic"); + +// /* Phase 1: produce and consume records to establish a share +// * session. auto.offset.reset=earliest (set by test_ctx_new) so +// * the share group starts from the beginning of the log. */ +// mock_produce(ctx.producer, topic, RD_KAFKA_PARTITION_UA, msgcnt); + +// /* Disable auto-create so the metadata handler does not recreate +// * the deleted topic, and disable the propagation wait so the +// * toppar is removed from the session immediately when metadata +// * confirms the topic is gone (instead of waiting 30 s). */ +// test_conf_init(&conf, NULL, 0); +// test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); +// test_conf_set(conf, "group.id", group); +// test_conf_set(conf, "share.acknowledgement.mode", "implicit"); +// test_conf_set(conf, "allow.auto.create.topics", "false"); +// test_conf_set(conf, "topic.metadata.propagation.max.ms", "0"); +// rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); +// TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); +// subscribe_one(rkshare, topic); + +// while (phase1_consumed < msgcnt && attempts++ < 30) { +// rcvd = 0; +// error = rd_kafka_share_poll(rkshare, 3000, +// rkmessages, +// &rcvd); +// if (error) { +// rd_kafka_error_destroy(error); +// continue; +// } +// for (j = 0; j < rcvd; j++) { +// if (!rkmessages[j]->err) +// phase1_consumed++; +// rd_kafka_message_destroy(rkmessages[j]); +// } +// } +// TEST_ASSERT(phase1_consumed == msgcnt, +// "Phase 1: expected %d records, got %d", msgcnt, +// phase1_consumed); + +// /* Start request tracking before topic deletion to capture +// * ShareFetch requests that get UNKNOWN_TOPIC_OR_PARTITION. */ +// rd_kafka_mock_start_request_tracking(ctx.mcluster); +// rd_kafka_mock_clear_requests(ctx.mcluster); + +// /* Delete the topic. The consumer's next ShareFetch will receive +// * UNKNOWN_TOPIC_OR_PARTITION per partition, which must be handled +// * silently (metadata refresh, no OP_CONSUMER_ERR). */ +// TEST_ASSERT(rd_kafka_mock_topic_delete(ctx.mcluster, topic) == +// RD_KAFKA_RESP_ERR_NO_ERROR, +// "delete topic"); + +// /* Phase 2: poll several times. OP_CONSUMER_ERR surfaces as a +// * non-NULL return from consume_batch; count any such return as +// * an error (silent handling requires zero). */ +// while (poll_cnt++ < 5) { +// rcvd = 0; +// error = rd_kafka_share_poll(rkshare, 1000, +// rkmessages, +// &rcvd); +// if (error) { +// error_cnt++; +// rd_kafka_error_destroy(error); +// continue; +// } +// for (j = 0; j < rcvd; j++) +// rd_kafka_message_destroy(rkmessages[j]); +// } +// TEST_ASSERT(error_cnt == 0, +// "expected 0 OP_CONSUMER_ERR after topic delete " +// "(UNKNOWN_TOPIC_OR_PARTITION must be silently handled), " +// "got %d", +// error_cnt); + +// /* Verify the error path fired. The background "keep fetching" loop +// * may send additional empty ShareFetch requests after the partition +// * leaves the session, so only a lower bound is meaningful here. */ +// share_fetch_cnt = test_mock_get_matching_request_cnt( +// ctx.mcluster, is_share_fetch_request, NULL); +// TEST_SAY("ShareFetch requests after topic delete: %" PRIusz "\n", +// share_fetch_cnt); +// TEST_ASSERT(share_fetch_cnt >= 1, +// "expected >= 1 ShareFetch after topic delete " +// "(error path must have fired), got %" PRIusz, +// share_fetch_cnt); + +// /* Verify a metadata refresh was triggered by the error. */ +// metadata_cnt = test_mock_get_matching_request_cnt( +// ctx.mcluster, is_metadata_request, NULL); +// TEST_SAY("Metadata requests after topic delete: %" PRIusz "\n", +// metadata_cnt); +// TEST_ASSERT(metadata_cnt >= 1, +// "expected >= 1 Metadata request after topic delete " +// "(metadata refresh must fire), got %" PRIusz, +// metadata_cnt); + +// rd_kafka_mock_stop_request_tracking(ctx.mcluster); + +// test_share_consumer_close(rkshare); +// test_share_destroy(rkshare); +// test_ctx_destroy(&ctx); + +// SUB_TEST_PASS(); +// } + +/* =================================================================== + * Regression test: records survive a leader-less partition transit. + * ------------------------------------------------------------------- + * Trigger: a partition becomes leader-less between two leader + * changes (leader -> -1 -> new_leader). The leader-less period + * makes the client briefly delegate the toppar to the :0/internal + * pseudo-broker. Previously, that pseudo-broker thread ran the + * regular-consumer fetch_decide path on the share-consumer rktp, + * promoting rktp_fetch_version above 0. Subsequent share-fetch + * parsing then stamped every parsed message rko with that + * promoted version, after which the version-outdated filter + * dropped them silently (because rktp_version had been bumped by + * the initial fetch_start barrier). + * + * Repro shape: + * 1. leader=1; produce phase 1; consume phase 1. + * 2. set leader=-1; consume again briefly so the consumer + * processes the metadata change and migrates the toppar to + * :0/internal. This is the step that historically promoted + * rktp_fetch_version on the share-consumer rktp. + * 3. wait 3 s for the bad state to settle. + * 4. set leader=2 but DO NOT consume yet — let the migration + * to broker 2 happen quietly. + * 5. produce phase 2 on the new leader. + * 6. consume phase 2 and verify every produced record reaches + * the application. + * + * Pre-fix: phase 2 consumed < phase 2 produced (parsed records + * dropped by the version filter). Post-fix: phase 2 fully drained. + * =================================================================== */ +static void +do_test_records_survive_leaderless_transit(rd_bool_t explicit_mode) { + test_ctx_t ctx; + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *drain_batch = NULL; + size_t drain_rcvd = 0; + char *topic; + const int part_cnt = 1; + const char *group = "sg-0183-leaderless-transit"; + int phase1_total = 0; + int phase2_total = 0; + int phase1_consumed = 0; + int phase2_consumed = 0; + int i; + + SUB_TEST_QUICK("%s", explicit_mode ? "explicit" : "implicit"); + + topic = rd_strdup(test_mk_topic_name("0183-leaderless-transit", 1)); + + ctx = test_ctx_new(3); + + TEST_ASSERT(rd_kafka_mock_topic_create(ctx.mcluster, topic, part_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + + /* All partitions start on broker 1. */ + for (i = 0; i < part_cnt; i++) + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, + topic, i, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set initial leader [%d] -> 1", i); + + srand(42); + for (i = 0; i < part_cnt; i++) + phase1_total += + mock_produce(ctx.producer, topic, i, 20 + (rand() % 11)); + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", ctx.bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", + explicit_mode ? "explicit" : "implicit"); + /* Short refresh interval so leader-change notifications + * land quickly. */ + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "2000"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "create share consumer"); + subscribe_one(rkshare, topic); + + /* Phase 1: consume initial batch from broker 1. This warms + * up the consumer's view of leader=1 for every partition. */ + phase1_consumed = consume_phase(rkshare, explicit_mode, rd_true, + phase1_total, 0, "Phase 1"); + TEST_ASSERT(phase1_consumed == phase1_total, + "Phase 1: expected %d, got %d", phase1_total, + phase1_consumed); + + /* First leader change: every partition becomes leader-less. + * The next consume call drives the share-fetch loop, which + * will hit broker 1, get NOT_LEADER_OR_FOLLOWER, refresh + * metadata, see leader_id=-1, and delegate the toppar to the + * :0/internal pseudo-broker — the exact bug-trigger. */ + for (i = 0; i < part_cnt; i++) + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, + topic, i, -1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader [%d] -> -1 (leader-less)", i); + + /* Mid consume_batch: drive the internal fetch cycle so the + * consumer hits broker 1 with SHAREFETCH, gets + * NOT_LEADER_OR_FOLLOWER, refreshes metadata, sees + * leader_id=-1, and delegates the rktp to the :0/internal + * pseudo-broker. No records expected (leader-less). */ + TEST_ASSERT(!rd_kafka_share_poll(rkshare, 500, &drain_batch), + "mid share_poll unexpected err"); + drain_rcvd = rd_kafka_messages_count(drain_batch); + TEST_ASSERT(drain_rcvd == 0, + "mid share_poll expected 0 records during the " + "leader-less window, got %" PRIusz, + drain_rcvd); + rd_kafka_messages_destroy(drain_batch); + drain_batch = NULL; + + /* Settle: pre-fix, :0/internal's broker_internal_serve loop + * runs consumer_toppars_serve here, which calls fetch_decide + * on the share-consumer rktp and promotes + * rktp_fetch_version. After this point, any parsed + * share-fetch message gets stamped with the promoted + * version and dropped by the version-outdated filter. */ + rd_usleep(3000 * 1000, NULL); + + /* Second leader change: leader=2. The fast-query timer + * armed by the NOT_LEADER response is still active and will + * fire within ~1s (capped by retry_backoff_max_ms), find + * leader=2, and clear LEADER_UNAVAIL. No consume here — let + * the metadata refresh happen on its own. */ + for (i = 0; i < part_cnt; i++) + TEST_ASSERT(rd_kafka_mock_partition_set_leader(ctx.mcluster, + topic, i, 2) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "set leader [%d] -> 2", i); + + /* Produce Phase 2 records on the new leader. */ + for (i = 0; i < part_cnt; i++) + phase2_total += + mock_produce(ctx.producer, topic, i, 20 + (rand() % 11)); + rd_kafka_flush(ctx.producer, 5000); + + /* Phase 2: every record produced post-transit must reach the + * application. Pre-fix this assertion failed because parsed + * SHAREFETCH messages were stamped with the promoted + * fetch_version and then dropped by the version-outdated + * filter against the bumped rktp_version. */ + phase2_consumed = consume_phase(rkshare, explicit_mode, rd_true, + phase2_total, 1500, "Phase 2"); + TEST_ASSERT(phase2_consumed == phase2_total, + "Phase 2: expected %d records after leader-less " + "transit + new leader, got %d. If pre-fix: parsed " + "records were silently dropped by the " + "rktp_version filter on the share-consumer path.", + phase2_total, phase2_consumed); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_ctx_destroy(&ctx); + rd_free(topic); + + SUB_TEST_PASS(); +} + + +int main_0183_share_consumer_leader_change_mock(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + test_timeout_set(120); + + test_shareack_leader_change_reduces_rpcs(); + test_partition_not_leader_or_follower_silent(); + /* test_partition_unknown_topic_silent(); */ + + do_test_records_survive_leaderless_transit(rd_false); + do_test_records_survive_leaderless_transit(rd_true); + + do_test_leader_change_consume_recovery(rd_false, rd_false, rd_false, + "implicit-no-refresh"); + do_test_leader_change_consume_recovery(rd_false, rd_false, rd_true, + "implicit-wait-refresh"); + do_test_leader_change_consume_recovery(rd_true, rd_false, rd_false, + "explicit-async-no-refresh"); + do_test_leader_change_consume_recovery(rd_true, rd_false, rd_true, + "explicit-async-wait-refresh"); + do_test_leader_change_consume_recovery(rd_true, rd_true, rd_false, + "explicit-sync-no-refresh"); + do_test_leader_change_consume_recovery(rd_true, rd_true, rd_true, + "explicit-sync-wait-refresh"); + return 0; +} \ No newline at end of file diff --git a/lib/librdkafka-2.15.0/tests/0184-share_consumer_topic_recreate.c b/lib/librdkafka-2.15.0/tests/0184-share_consumer_topic_recreate.c new file mode 100644 index 00000000000..54a572d1fd8 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0184-share_consumer_topic_recreate.c @@ -0,0 +1,1356 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "testshared.h" +#include "rdkafka.h" +#include "../src/rdkafka_proto.h" + +#define PARTITION_CNT 3 +#define MSGS_PER_PARTITION 10 +#define MSGS_PER_PHASE (PARTITION_CNT * MSGS_PER_PARTITION) +#define BATCH_SIZE 1024 + + +static rd_kafka_t *common_producer; +static rd_kafka_t *common_admin; + +/**************************************************************************** + * Helpers + ****************************************************************************/ + +/** + * @brief Produce MSGS_PER_PARTITION framework-encoded messages (NULL + * payload => librdkafka test helper auto-encodes + * testid+partition+msgid into payload and key) to each of the + * first \p partition_cnt partitions of \p topic. + * + * Per-partition msgids run [0, MSGS_PER_PARTITION), so msgver_verify_part + * can later assert exact-range coverage independently per partition. + */ +static void +produce_phase(const char *topic, uint64_t testid, int32_t partition_cnt) { + int32_t p; + for (p = 0; p < partition_cnt; p++) { + test_produce_msgs2(common_producer, topic, testid, p, + 0 /*msg_base*/, MSGS_PER_PARTITION, + NULL /*payload*/, 0); + } +} + +/** + * @brief Fetch the broker-assigned topic_id for \p topic via the + * DescribeTopics admin API. + * + * @returns A newly-allocated Uuid that the caller must destroy with + * rd_kafka_Uuid_destroy(). + * + * On any failure (topic missing, admin timeout, etc.) the test is failed. + */ +static rd_kafka_Uuid_t *fetch_topic_id(const char *topic) { + rd_kafka_queue_t *q; + rd_kafka_TopicCollection_t *tc; + rd_kafka_event_t *rkev; + const rd_kafka_DescribeTopics_result_t *res; + const rd_kafka_TopicDescription_t **descs; + rd_kafka_Uuid_t *topic_id; + size_t desc_cnt = 0; + const char *topics_arr[1]; + + topics_arr[0] = topic; + + q = rd_kafka_queue_new(common_admin); + tc = rd_kafka_TopicCollection_of_topic_names(topics_arr, 1); + + rd_kafka_DescribeTopics(common_admin, tc, NULL /*options*/, q); + + rkev = rd_kafka_queue_poll(q, 30 * 1000); + TEST_ASSERT(rkev != NULL, "DescribeTopics(%s) timed out", topic); + TEST_ASSERT(rd_kafka_event_error(rkev) == RD_KAFKA_RESP_ERR_NO_ERROR, + "DescribeTopics(%s) failed: %s", topic, + rd_kafka_event_error_string(rkev)); + + res = rd_kafka_event_DescribeTopics_result(rkev); + descs = rd_kafka_DescribeTopics_result_topics(res, &desc_cnt); + TEST_ASSERT(desc_cnt == 1, "Expected 1 topic description, got %zu", + desc_cnt); + TEST_ASSERT( + rd_kafka_error_code(rd_kafka_TopicDescription_error(descs[0])) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Topic %s description error: %s", topic, + rd_kafka_error_string(rd_kafka_TopicDescription_error(descs[0]))); + + topic_id = + rd_kafka_Uuid_copy(rd_kafka_TopicDescription_topic_id(descs[0])); + + rd_kafka_event_destroy(rkev); + rd_kafka_TopicCollection_destroy(tc); + rd_kafka_queue_destroy(q); + + return topic_id; +} + +/** + * @brief Assert that \p id_before and \p id_after represent two different + * topic instances; fail the test otherwise. + */ +static void assert_topic_id_changed(const rd_kafka_Uuid_t *id_before, + const rd_kafka_Uuid_t *id_after) { + TEST_ASSERT(strcmp(rd_kafka_Uuid_base64str(id_before), + rd_kafka_Uuid_base64str(id_after)) != 0, + "Recreated topic_id (%s) matches the original; " + "broker did not actually recreate the topic", + rd_kafka_Uuid_base64str(id_before)); +} + +/** + * @brief Consume from a share consumer into \p mv until \p exp_cnt messages + * matching mv->testid have been collected, or \p timeout_ms elapses. + * + * Every consumed message is ACCEPT-ack'd. On every message we assert that + * delivery_count == 1 (no broker-side redelivery occurred for this run). + * + * @returns the count of matching messages collected into \p mv. + */ +static int consume_into_msgver(rd_kafka_share_t *rkshare, + test_msgver_t *mv, + int exp_cnt, + int timeout_ms) { + rd_kafka_messages_t *batch = NULL; + rd_ts_t deadline = test_clock() + (rd_ts_t)timeout_ms * 1000; + rd_kafka_t *rk = test_share_consumer_get_rk(rkshare); + + while (mv->msgcnt < exp_cnt && test_clock() < deadline) { + rd_kafka_error_t *err; + size_t rcvd = 0; + size_t i; + + err = rd_kafka_share_poll(rkshare, 500, &batch); + if (err) { + TEST_SAY("share_poll error: %s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + continue; + } + + rcvd = rd_kafka_messages_count(batch); + + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, i); + + if (m->err) { + TEST_SAY( + "Consumer event: %s " + "(topic=%s partition=%" PRId32 ")\n", + rd_kafka_err2str(m->err), + m->rkt ? rd_kafka_topic_name(m->rkt) + : "(none)", + m->partition); + continue; + } + + TEST_ASSERT( + rd_kafka_message_delivery_count(m) == 1, + "Unexpected redelivery: delivery_count=%d on " + "topic=%s partition=%" PRId32 " offset=%" PRId64, + rd_kafka_message_delivery_count(m), + rd_kafka_topic_name(m->rkt), m->partition, + m->offset); + + /* msgver auto-filters by mv->testid; messages from + * the other phase (or untagged) are silently + * dropped. */ + test_msgver_add_msg(rk, mv, m); + rd_kafka_share_acknowledge(rkshare, m); + } + + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + return mv->msgcnt; +} + +/** + * @brief Delete a topic, wait for the deletion to settle, then recreate + * with the same name and \p partition_cnt partitions. The new + * instance will have a fresh topic_id on the broker. + */ +static void recreate_topic(const char *topic, int partition_cnt) { + TEST_SAY("Deleting topic %s\n", topic); + test_delete_topic(common_admin, topic); + + /* DeleteTopics is async on the broker; the controller may still + * be removing log segments when we issue CreateTopics. Without + * this pause, the broker frequently lets the CreateTopics request + * sit until DeleteTopics finishes, which can blow past the + * admin-op timeout. The same 5s pause pattern is used by + * 0107-topic_recreate. */ + rd_sleep(5); + + TEST_SAY("Recreating topic %s with %d partition(s)\n", topic, + partition_cnt); + test_create_topic_wait_exists(common_admin, topic, partition_cnt, -1, + 60 * 1000); +} + +/** + * @brief Build a share consumer such that the periodic metadata refresh + * is the dominant channel for discovering topic_id changes: + * - topic.metadata.refresh.interval.ms is set to 500ms, well + * below the broker-dictated ShareGroupHeartbeat interval + * (typically 5s), so the refresh timer is virtually + * guaranteed to fire between heartbeats during the + * delete/recreate window; + */ +static rd_kafka_share_t *create_consumer_md_first(const char *group_id) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500"); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + + return rkshare; +} + +/** + * @brief Build a share consumer such that the ShareGroupHeartbeat is + * the dominant channel for discovering topic_id changes; the + * default topic.metadata.refresh.interval.ms is 5min, so the + * periodic MD refresh effectively does not fire during a test. + * + * Same signature as create_consumer_md_first so both can be passed to + * do_test_recreate_two_phase as a builder function pointer. + */ +static rd_kafka_share_t *create_consumer_hb_first(const char *group_id) { + return test_create_share_consumer(group_id, "explicit"); +} + +/** + * @brief Background-thread payload that deletes and recreates a topic + * on a mock cluster after a short sleep. + * + * Used by recreate-during-close mock tests so the recreate lands in the + * middle of an artificially-delayed broker request on the main thread. + */ +struct recreate_thread_args { + rd_kafka_mock_cluster_t *mcluster; + const char *topic; + int32_t partition_cnt; + int sleep_ms; +}; + +static int recreate_thread_main(void *p) { + struct recreate_thread_args *args = p; + rd_usleep((int64_t)args->sleep_ms * 1000, NULL); + TEST_SAY("[recreate-thread] Deleting topic %s\n", args->topic); + TEST_ASSERT(rd_kafka_mock_topic_delete(args->mcluster, args->topic) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "[recreate-thread] mock_topic_delete failed"); + TEST_SAY("[recreate-thread] Recreating topic %s with %" PRId32 + " partition(s)\n", + args->topic, args->partition_cnt); + TEST_ASSERT(rd_kafka_mock_topic_create(args->mcluster, args->topic, + args->partition_cnt, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "[recreate-thread] mock_topic_create failed"); + return 0; +} + +/** + * @brief Background producer thread: produces one record every + * producer_period_ms with payload ":", where + * phase is whatever the controlling thread has most recently + * written into args->phase. Tolerates produce errors (the + * topic may not exist mid-test). + * + * + * Used by the survives-delete-and-recreate test to keep produce + * activity steady through the delete window. + */ +struct producer_thread_args { + rd_kafka_t *producer; + const char *topic; + int period_ms; + char phase[16]; /* "before" / "during" / "after" */ + rd_bool_t stop; +}; + +static int producer_thread_main(void *p) { + struct producer_thread_args *args = p; + int seq = 0; + while (!args->stop) { + char payload[64]; + rd_snprintf(payload, sizeof(payload), "%s:%d", args->phase, + seq++); + /* Errors during the delete window are expected; the + * test tolerates them silently. */ + (void)rd_kafka_producev( + args->producer, RD_KAFKA_V_TOPIC(args->topic), + RD_KAFKA_V_VALUE(payload, strlen(payload)), + RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_END); + rd_usleep((int64_t)args->period_ms * 1000, NULL); + } + return 0; +} + +/**************************************************************************** + * Test Cases + ****************************************************************************/ + +/** + * @brief Generic two-phase recreate test: produce + consume + verify + * a "before" phase, recreate the topic (possibly with a different + * partition count), then produce + consume + verify an "after" + * phase against the new topic instance. + * + * 1. Subscribe; produce MSGS_PER_PHASE framework-encoded messages + * tagged with testid_before; consume them all via a "before" msgver + * and assert exact range coverage and zero redelivery. + * 2. Delete + recreate the topic with \p after_partition_cnt partitions. + * The broker assigns a fresh topic_id. Whichever of (HB, periodic + * metadata refresh) the \p ctor builder favors will carry the + * change to the client first. + * 3. Produce after_partition_cnt * MSGS_PER_PARTITION records tagged + * with testid_after; consume them all via an "after" msgver and + * assert exact range coverage, zero redelivery, and (implicitly + * via testid filtering) zero stale "before" messages. + */ +static void do_test_recreate_two_phase(const char *label, + const char *group_id, + rd_kafka_share_t *(*ctor)(const char *), + int32_t after_partition_cnt) { + const char *topic; + rd_kafka_share_t *rkshare; + test_msgver_t mv_before, mv_after; + uint64_t testid_before, testid_after; + rd_kafka_Uuid_t *id_before, *id_after; + int got; + int32_t p; + const int msgs_per_phase_after = + after_partition_cnt * MSGS_PER_PARTITION; + + SUB_TEST_QUICK("%s", label); + + testid_before = test_id_generate(); + testid_after = test_id_generate(); + + topic = test_mk_topic_name(label, 1); + + test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1, + 60 * 1000); + + id_before = fetch_topic_id(topic); + TEST_SAY("Initial topic_id: %s\n", rd_kafka_Uuid_base64str(id_before)); + + test_share_set_auto_offset_reset(group_id, "earliest"); + + rkshare = ctor(group_id); + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* ---- Phase 1: pre-recreate ---- */ + TEST_SAY( + "Phase 1: producing %d msgs/partition x %d partitions " + "(testid=%" PRIu64 ")\n", + MSGS_PER_PARTITION, PARTITION_CNT, testid_before); + produce_phase(topic, testid_before, PARTITION_CNT); + + test_msgver_init(&mv_before, testid_before); + got = + consume_into_msgver(rkshare, &mv_before, MSGS_PER_PHASE, 30 * 1000); + TEST_ASSERT(got == MSGS_PER_PHASE, + "Phase 1: expected %d msgs matching testid_before, " + "got %d", + MSGS_PER_PHASE, got); + for (p = 0; p < PARTITION_CNT; p++) + test_msgver_verify_part("phase1-before-part", &mv_before, + TEST_MSGVER_ALL_PART, topic, p, + 0 /*msg_base*/, MSGS_PER_PARTITION); + test_msgver_clear(&mv_before); + + /* ---- Recreate ---- */ + recreate_topic(topic, after_partition_cnt); + + id_after = fetch_topic_id(topic); + TEST_SAY("Recreated topic_id: %s\n", rd_kafka_Uuid_base64str(id_after)); + assert_topic_id_changed(id_before, id_after); + + /* ---- Phase 2: post-recreate ---- */ + TEST_SAY("Phase 2: producing %d msgs/partition x %" PRId32 + " partitions on recreated topic (testid=%" PRIu64 ")\n", + MSGS_PER_PARTITION, after_partition_cnt, testid_after); + produce_phase(topic, testid_after, after_partition_cnt); + + test_msgver_init(&mv_after, testid_after); + got = consume_into_msgver(rkshare, &mv_after, msgs_per_phase_after, + 60 * 1000); + TEST_ASSERT(got == msgs_per_phase_after, + "Phase 2: expected %d msgs matching testid_after, " + "got %d (post-recreate)", + msgs_per_phase_after, got); + for (p = 0; p < after_partition_cnt; p++) + test_msgver_verify_part("phase2-after-part", &mv_after, + TEST_MSGVER_ALL_PART, topic, p, + 0 /*msg_base*/, MSGS_PER_PARTITION); + test_msgver_clear(&mv_after); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_Uuid_destroy(id_before); + rd_kafka_Uuid_destroy(id_after); + + SUB_TEST_PASS(); +} + +/** + * @brief Stress test: many rapid topic recreations interleaved with + * random produce activity and varying partition counts. + * + * Each chaos cycle picks (uniformly at random): + * - A short pre-recreate sleep in [200, 1500] ms, to let the + * consumer's state advance unpredictably between cycles. + * - Whether to produce 0..5 records to each partition of the + * current topic instance before the recreate; if any are + * produced, they use a per-cycle testid that the test does not + * track (we don't require these records to be received). + * - A new partition count from {2, 3, 4} for the recreated topic, + * exercising shrink, no-change, and grow paths from one cycle to + * the next. + * + * After chaos_cycles iterations the test produces a final batch of + * MSGS_PER_PARTITION records to every partition of whatever topic + * instance is currently live (partition count is whatever chaos left + * it at) and asserts: + * - The final batch is fully consumed (per-partition exact-range). + * - All final-batch records have delivery_count == 1. + * - The broker-assigned topic_id changed end-to-end. + */ +static void do_test_recreate_chaos(void) { + const int chaos_cycles = 10; + const int chaos_sleep_min_ms = 200; + const int chaos_sleep_max_ms = 1500; + const int chaos_max_records_per_part = 5; + const int32_t chaos_partition_choices[] = {2, 3, 4}; + const int chaos_partition_choices_cnt = + sizeof(chaos_partition_choices) / + sizeof(chaos_partition_choices[0]); + const char *topic; + const char *group_id = "0184-share-recreate-chaos"; + rd_kafka_share_t *rkshare; + test_msgver_t mv_warmup, mv_final; + uint64_t testid_warmup, testid_final, seed_testid; + rd_kafka_Uuid_t *id_initial, *id_final; + int32_t current_partition_cnt = PARTITION_CNT; + int cycle; + int got; + int32_t p; + int final_msgs; + + SUB_TEST_QUICK(); + + seed_testid = test_id_generate(); + srand((unsigned)seed_testid); + TEST_SAY("Chaos seed (from testid): %" PRIu64 "\n", seed_testid); + + testid_warmup = test_id_generate(); + testid_final = test_id_generate(); + + topic = test_mk_topic_name("0184-recreate-chaos", 1); + + test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1, + 60 * 1000); + id_initial = fetch_topic_id(topic); + TEST_SAY("Initial topic_id: %s\n", rd_kafka_Uuid_base64str(id_initial)); + + test_share_set_auto_offset_reset(group_id, "earliest"); + + /* HB-first regime: long MD refresh so HB drives discovery + * across recreate cycles. */ + rkshare = test_create_share_consumer(group_id, "explicit"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Warmup: drive the consumer to a fully-active state before + * chaos begins. Produces MSGS_PER_PARTITION to every + * partition; consumes them all. */ + TEST_SAY( + "Warmup: producing %d msgs/partition x %d partitions " + "(testid=%" PRIu64 ")\n", + MSGS_PER_PARTITION, PARTITION_CNT, testid_warmup); + produce_phase(topic, testid_warmup, PARTITION_CNT); + + test_msgver_init(&mv_warmup, testid_warmup); + got = + consume_into_msgver(rkshare, &mv_warmup, MSGS_PER_PHASE, 30 * 1000); + TEST_ASSERT(got == MSGS_PER_PHASE, "Warmup: expected %d msgs, got %d", + MSGS_PER_PHASE, got); + test_msgver_clear(&mv_warmup); + + /* Chaos loop. */ + for (cycle = 0; cycle < chaos_cycles; cycle++) { + int sleep_ms; + int32_t next_partition_cnt; + int do_produce; + int n_per_part; + + sleep_ms = + chaos_sleep_min_ms + + (rand() % (chaos_sleep_max_ms - chaos_sleep_min_ms + 1)); + next_partition_cnt = + chaos_partition_choices[rand() % + chaos_partition_choices_cnt]; + do_produce = rand() % 2; + n_per_part = do_produce + ? (rand() % (chaos_max_records_per_part + 1)) + : 0; + + TEST_SAY( + "[chaos cycle %d/%d] sleep=%dms, produce=%d " + "msg(s)/partition (cur_partitions=%" PRId32 + "), next_partition_cnt=%" PRId32 "\n", + cycle + 1, chaos_cycles, sleep_ms, n_per_part, + current_partition_cnt, next_partition_cnt); + + rd_usleep((int64_t)sleep_ms * 1000, NULL); + + /* Optional mid-cycle produce. Uses a throwaway testid + * so these records are silently ignored on consume — + * they may or may not arrive depending on what state + * the consumer is in. */ + if (n_per_part > 0) { + uint64_t cycle_testid = test_id_generate(); + int32_t cp; + for (cp = 0; cp < current_partition_cnt; cp++) { + test_produce_msgs2(common_producer, topic, + cycle_testid, cp, + 0 /*msg_base*/, n_per_part, + NULL /*payload*/, 0); + } + } + + recreate_topic(topic, next_partition_cnt); + current_partition_cnt = next_partition_cnt; + + /* Force a producer-side metadata refresh + * so its view matches the broker before we try to + * produce again. */ + test_wait_topic_exists(common_producer, topic, 60 * 1000); + } + + /* Settle: produce a final batch we will require to be fully + * consumed, against whatever topic instance chaos left us + * with. */ + TEST_SAY("Settle: producing %d msgs/partition x %" PRId32 + " partitions (testid=%" PRIu64 ")\n", + MSGS_PER_PARTITION, current_partition_cnt, testid_final); + produce_phase(topic, testid_final, current_partition_cnt); + + final_msgs = current_partition_cnt * MSGS_PER_PARTITION; + + test_msgver_init(&mv_final, testid_final); + got = consume_into_msgver(rkshare, &mv_final, final_msgs, 60 * 1000); + TEST_ASSERT(got == final_msgs, + "Settle: expected %d msgs matching testid_final, " + "got %d after %d chaos cycles", + final_msgs, got, chaos_cycles); + for (p = 0; p < current_partition_cnt; p++) + test_msgver_verify_part("chaos-final-part", &mv_final, + TEST_MSGVER_ALL_PART, topic, p, + 0 /*msg_base*/, MSGS_PER_PARTITION); + test_msgver_clear(&mv_final); + + /* Verify chaos actually moved the topic_id. */ + id_final = fetch_topic_id(topic); + TEST_SAY("Final topic_id: %s\n", rd_kafka_Uuid_base64str(id_final)); + assert_topic_id_changed(id_initial, id_final); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_Uuid_destroy(id_initial); + rd_kafka_Uuid_destroy(id_final); + + SUB_TEST_PASS(); +} + +/** + * @brief Subscribed consumer survives topic delete + recreate-with- + * different-partition-count while a producer is actively + * producing across the whole window. + * + * Differs from the other recreate tests in that this one keeps a + * concurrent producer thread running through the entire delete window. + * + * Assertions: + * - At least one "before" record consumed (consumer was healthy). + * - At least one "after" record consumed (consumer recovered). + * - "During" records may or may not be consumed; no assertion. + * - Final topic_id differs from the initial topic_id (broker + * actually recreated the topic). + */ +static void do_test_recreate_survives_concurrent_producer(void) { + const char *topic; + const char *group_id = "0184-share-recreate-concurrent-producer"; + const int32_t partition_cnt_a = 3; + const int32_t partition_cnt_b = 5; + const int producer_period_ms = 100; /* ~10 msgs/sec */ + const int phase_duration_ms = 10000; + rd_kafka_share_t *rkshare; + rd_kafka_messages_t *batch = NULL; + struct producer_thread_args producer_args; + thrd_t producer_thrd; + rd_kafka_Uuid_t *id_initial, *id_final; + int before_cnt = 0, during_cnt = 0, after_cnt = 0; + rd_ts_t consume_deadline; + + SUB_TEST_QUICK(); + + topic = test_mk_topic_name("0184-recreate-concurrent-producer", 1); + + test_create_topic_wait_exists(common_admin, topic, partition_cnt_a, -1, + 60 * 1000); + id_initial = fetch_topic_id(topic); + TEST_SAY("Initial topic_id: %s (partition_cnt=%" PRId32 ")\n", + rd_kafka_Uuid_base64str(id_initial), partition_cnt_a); + + test_share_set_auto_offset_reset(group_id, "earliest"); + + rkshare = test_create_share_consumer(group_id, "implicit"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* Start background producer in "before" phase. */ + producer_args.producer = common_producer; + producer_args.topic = topic; + producer_args.period_ms = producer_period_ms; + producer_args.stop = rd_false; + rd_snprintf(producer_args.phase, sizeof(producer_args.phase), "before"); + + TEST_ASSERT(thrd_create(&producer_thrd, producer_thread_main, + &producer_args) == thrd_success, + "thrd_create failed"); + + /* --- Phase: BEFORE. + * Consume for a few seconds while producer thread feeds the + * topic. Count "before:" records to confirm the pipeline is + * live. */ + consume_deadline = test_clock() + (rd_ts_t)phase_duration_ms * 1000; + while (test_clock() < consume_deadline) { + rd_kafka_error_t *err; + size_t rcvd = 0; + size_t i; + err = rd_kafka_share_poll(rkshare, 200, &batch); + if (err) { + TEST_SAY( + "Phase BEFORE: share_poll error: " + "%s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, i); + if (!m->err && m->payload && m->len >= 7 && + !strncmp((const char *)m->payload, "before:", 7)) + before_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_SAY("Phase BEFORE: consumed %d \"before:\" record(s)\n", + before_cnt); + + /* --- Transition: enter DURING. Delete the topic but keep the + * producer running. Most produces in this window will fail with + * Unknown topic-type errors; that's expected. */ + TEST_SAY("Phase transition: entering DURING (delete topic)\n"); + rd_snprintf(producer_args.phase, sizeof(producer_args.phase), "during"); + test_delete_topic(common_admin, topic); + + /* Drain the consumer briefly during the delete window. Most + * fetches will return errors. We don't track "during:" counts + * for an assertion; just log. */ + consume_deadline = test_clock() + (rd_ts_t)phase_duration_ms * 1000; + while (test_clock() < consume_deadline) { + rd_kafka_error_t *err; + size_t rcvd = 0; + size_t i; + err = rd_kafka_share_poll(rkshare, 200, &batch); + if (err) { + TEST_SAY( + "Phase DURING: share_poll error: " + "%s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, i); + if (!m->err && m->payload && m->len >= 7 && + !strncmp((const char *)m->payload, "during:", 7)) + during_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_SAY( + "Phase DURING: consumed %d \"during:\" record(s) " + "(no assertion on this count)\n", + during_cnt); + + /* --- Transition: recreate with a different partition count + * and switch producer to "after". */ + TEST_SAY("Recreating topic with partition_cnt=%" PRId32 " (was %" PRId32 + ")\n", + partition_cnt_b, partition_cnt_a); + rd_sleep(5); /* let delete settle on the broker */ + test_create_topic_wait_exists(common_admin, topic, partition_cnt_b, -1, + 60 * 1000); + + id_final = fetch_topic_id(topic); + TEST_SAY("Recreated topic_id: %s (partition_cnt=%" PRId32 ")\n", + rd_kafka_Uuid_base64str(id_final), partition_cnt_b); + + /* Force the producer to refresh its metadata for the recreated + * topic. + */ + test_wait_topic_exists(common_producer, topic, 60 * 1000); + + rd_snprintf(producer_args.phase, sizeof(producer_args.phase), "after"); + + /* --- Phase: AFTER. + * Consume for a longer window to give the consumer time to + * reconcile the new topic_id and start delivering "after:" + * records. */ + consume_deadline = test_clock() + 60 * 1000 * 1000; + while (after_cnt == 0 && test_clock() < consume_deadline) { + rd_kafka_error_t *err; + size_t rcvd = 0; + size_t i; + err = rd_kafka_share_poll(rkshare, 500, &batch); + if (err) { + TEST_SAY( + "Phase AFTER: share_poll error: " + "%s\n", + rd_kafka_error_string(err)); + rd_kafka_error_destroy(err); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, i); + if (!m->err && m->payload && m->len >= 6 && + !strncmp((const char *)m->payload, "after:", 6)) + after_cnt++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_SAY("Phase AFTER: consumed %d \"after:\" record(s)\n", after_cnt); + + /* Stop and join the producer thread before any assertion + * runs. + */ + producer_args.stop = rd_true; + thrd_join(producer_thrd, NULL); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + + TEST_ASSERT(before_cnt > 0, + "Expected at least one \"before:\" record to be " + "consumed"); + assert_topic_id_changed(id_initial, id_final); + TEST_ASSERT(after_cnt > 0, + "Consumer did not recover after recreate: zero " + "\"after:\" records seen within 60s"); + + rd_kafka_Uuid_destroy(id_initial); + rd_kafka_Uuid_destroy(id_final); + + SUB_TEST_PASS(); +} + +/** + * @brief Topic ID changes server-side while the consumer is in + * the middle of close(); verify close completes cleanly. + * + * Assertions: + * - close() completes within a reasonable time + */ +static void test_recreate_during_close(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_error_t *error; + const char *topic = "0184-recreate-during-close"; + const char *group = "0184-share-recreate-during-close"; + const int n_msgs = 10; + const int delay_ms = 5000; + rd_kafka_message_t *rkmessages[64]; + rd_kafka_messages_t *batches[64]; + size_t batches_cnt = 0; + struct recreate_thread_args thread_args; + thrd_t recreate_thrd; + rd_kafka_conf_t *conf; + size_t rcvd = 0; + int attempts = 0; + int max_attempt = 30; + int i; + rd_ts_t t_start, t_elapsed_ms; + const int64_t close_upper_bound_ms = delay_ms + 2000; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to create mock topic"); + + TEST_SAY("Producing %d messages to topic %s\n", n_msgs, topic); + test_produce_msgs_easy_v(topic, 0, 0, 0, n_msgs, 16, + "bootstrap.servers", bootstraps, NULL); + + TEST_SAY("Creating share consumer with explicit ack mode\n"); + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + TEST_SAY("Consuming up to %d messages\n", n_msgs); + while (rcvd < (size_t)n_msgs && attempts < max_attempt) { + rd_kafka_messages_t *batch = NULL; + size_t batch_rcvd; + size_t k; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + TEST_SAY("Consume attempt %d: %s\n", attempts, + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + attempts++; + continue; + } + batch_rcvd = rd_kafka_messages_count(batch); + if (batch_rcvd > 0 && + batches_cnt < sizeof(batches) / sizeof(batches[0]) && + rcvd + batch_rcvd <= + sizeof(rkmessages) / sizeof(rkmessages[0])) { + for (k = 0; k < batch_rcvd; k++) + rkmessages[rcvd + k] = + rd_kafka_messages_get(batch, k); + rcvd += batch_rcvd; + batches[batches_cnt++] = batch; + } else { + rd_kafka_messages_destroy(batch); + } + attempts++; + } + TEST_ASSERT(rcvd == (size_t)n_msgs, + "Expected to consume %d messages, got %zu", n_msgs, rcvd); + + TEST_SAY("Staging acks for all %zu messages\n", rcvd); + for (i = 0; i < (int)rcvd; i++) { + rd_kafka_resp_err_t ack_err = + rd_kafka_share_acknowledge(rkshare, rkmessages[i]); + TEST_ASSERT(ack_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "ack %d failed: %s", i, rd_kafka_err2str(ack_err)); + } + + TEST_SAY("Injecting %dms delay on the next ShareAcknowledge response\n", + delay_ms); + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareAcknowledge, 1, + RD_KAFKA_RESP_ERR_NO_ERROR, + delay_ms) == RD_KAFKA_RESP_ERR_NO_ERROR, + "Failed to inject ShareAcknowledge delay"); + + thread_args.mcluster = mcluster; + thread_args.topic = topic; + thread_args.partition_cnt = 1; + thread_args.sleep_ms = 1000; + TEST_ASSERT(thrd_create(&recreate_thrd, recreate_thread_main, + &thread_args) == thrd_success, + "thrd_create failed"); + + TEST_SAY( + "Calling close(); broker will hold ack for %dms while " + "background thread recreates the topic\n", + delay_ms); + t_start = test_clock(); + error = rd_kafka_share_consumer_close(rkshare); + t_elapsed_ms = (test_clock() - t_start) / 1000; + TEST_SAY("close() returned in %" PRId64 " ms\n", t_elapsed_ms); + + if (error) { + TEST_SAY("close() returned error: %s\n", + rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + } + + TEST_ASSERT(t_elapsed_ms <= close_upper_bound_ms, + "close() took %" PRId64 " ms, expected <= %" PRId64 + " ms (broker delay + overhead)", + t_elapsed_ms, close_upper_bound_ms); + + thrd_join(recreate_thrd, NULL); + + for (i = 0; i < (int)batches_cnt; i++) + rd_kafka_messages_destroy(batches[i]); + + test_share_destroy(rkshare); + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +/* =================================================================== + * Log callback: counts the "no partitions are assigned" fetch-stall + * log emitted while the share consumer's assignment is empty. + * =================================================================== */ +static void recreate_stall_log_cb(const rd_kafka_t *rk, + int level, + const char *fac, + const char *buf) { + rd_atomic32_t *cnt = rd_kafka_opaque(rk); + if (cnt && !strcmp(fac, "FETCHMORE") && + strstr(buf, "no partitions are assigned")) + rd_atomic32_add(cnt, 1); +} + +/** + * @brief Delete the consumer's only topic so the broker revokes its + * assignment to empty, verify the consumer reports the + * no-partitions fetch stall, then recreate the topic and verify + * records flow again. + * + * Reproduces the observed delete/recreate latency: with no partitions + * assigned the consumer keeps wanting to fetch and emits the + * rate-limited "Fetch stalled: ... no partitions are assigned" log + * until the recreated topic (fresh topic_id) is reassigned. + */ +static void do_test_recreate_stall_then_recover(void) { + const char *topic; + const char *group = "0184-share-recreate-stall"; + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_atomic32_t stall_cnt; + test_msgver_t mv_before, mv_after; + uint64_t testid_before, testid_after; + rd_kafka_Uuid_t *id_before, *id_after; + rd_kafka_messages_t *batch = NULL; + rd_ts_t deadline; + int got, stalls; + char errstr[512]; + + SUB_TEST_QUICK(); + + rd_atomic32_init(&stall_cnt, 0); + testid_before = test_id_generate(); + testid_after = test_id_generate(); + + topic = test_mk_topic_name("0184-recreate-stall", 1); + test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1, + 60 * 1000); + + id_before = fetch_topic_id(topic); + test_share_set_auto_offset_reset(group, "earliest"); + + /* Fast metadata refresh + debug=consumer so the no-partitions + * stall log is emitted and observable via the log callback. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "500"); + test_conf_set(conf, "debug", "consumer"); + rd_kafka_conf_set_log_cb(conf, recreate_stall_log_cb); + rd_kafka_conf_set_opaque(conf, &stall_cnt); + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* ---- Phase 1: consume the original generation ---- */ + produce_phase(topic, testid_before, PARTITION_CNT); + test_msgver_init(&mv_before, testid_before); + got = + consume_into_msgver(rkshare, &mv_before, MSGS_PER_PHASE, 30 * 1000); + TEST_ASSERT(got == MSGS_PER_PHASE, "Phase 1: expected %d msgs, got %d", + MSGS_PER_PHASE, got); + test_msgver_clear(&mv_before); + + /* ---- Delete: the broker revokes the assignment to empty ---- */ + TEST_SAY("Deleting topic %s; expecting a no-partitions fetch stall\n", + topic); + test_delete_topic(common_admin, topic); + + /* Poll until the no-partitions stall is logged. The log is + * rate-limited, so a single occurrence confirms the path. */ + deadline = test_clock() + (rd_ts_t)60 * 1000 * 1000; + while (rd_atomic32_get(&stall_cnt) < 1 && test_clock() < deadline) { + rd_kafka_error_t *error = + rd_kafka_share_poll(rkshare, 500, &batch); + if (error) { + rd_kafka_error_destroy(error); + } else { + /* Ack any straggler so the next poll can proceed + * in explicit mode. */ + size_t i, n = rd_kafka_messages_count(batch); + for (i = 0; i < n; i++) + rd_kafka_share_acknowledge( + rkshare, rd_kafka_messages_get(batch, i)); + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + stalls = rd_atomic32_get(&stall_cnt); + TEST_SAY("\"no partitions assigned\" stall log count: %d\n", stalls); + TEST_ASSERT(stalls >= 1, + "expected the no-partitions stall to be logged, got %d", + stalls); + + /* ---- Recreate (fresh topic_id): the broker reassigns ---- */ + rd_sleep(5); /* let DeleteTopics settle, as in recreate_topic() */ + test_create_topic_wait_exists(common_admin, topic, PARTITION_CNT, -1, + 60 * 1000); + id_after = fetch_topic_id(topic); + assert_topic_id_changed(id_before, id_after); + + /* ---- Phase 2: records flow again ---- */ + produce_phase(topic, testid_after, PARTITION_CNT); + test_msgver_init(&mv_after, testid_after); + got = + consume_into_msgver(rkshare, &mv_after, MSGS_PER_PHASE, 60 * 1000); + TEST_ASSERT(got == MSGS_PER_PHASE, + "Phase 2: expected %d msgs after recreate, got %d", + MSGS_PER_PHASE, got); + test_msgver_clear(&mv_after); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_Uuid_destroy(id_before); + rd_kafka_Uuid_destroy(id_after); + + SUB_TEST_PASS(); +} + +int main_0184_share_consumer_topic_recreate(int argc, char **argv) { + /* Topic deletion is not supported against Windows brokers. */ + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) { + TEST_SKIP("Topic deletion not supported on Windows brokers\n"); + return 0; + } + + test_timeout_set(300); + + common_producer = test_create_producer(); + common_admin = test_create_producer(); + + do_test_recreate_two_phase("0184-recreate-hb-first", + "0184-share-recreate-hb-first", + create_consumer_hb_first, PARTITION_CNT); + do_test_recreate_two_phase("0184-recreate-md-first", + "0184-share-recreate-md-first", + create_consumer_md_first, PARTITION_CNT); + do_test_recreate_two_phase("0184-recreate-shrink-hb-first", + "0184-share-recreate-shrink-hb-first", + create_consumer_hb_first, 2); + do_test_recreate_two_phase("0184-recreate-shrink-md-first", + "0184-share-recreate-shrink-md-first", + create_consumer_md_first, 2); + do_test_recreate_chaos(); + do_test_recreate_survives_concurrent_producer(); + do_test_recreate_stall_then_recover(); + + rd_kafka_destroy(common_admin); + rd_kafka_destroy(common_producer); + + return 0; +} + +/** + * @brief Same-name topic recreate where a single consumer owns the same + * (name, partition) tuples in both generations. + * + * The cooperative-assignment reconciler in rd_kafka_cgrp.c builds set + * subtraction maps keyed by (topic_name, partition), so OLD-id and + * NEW-id entries for the same (name, partition) collide in the diff. + * When a single consumer owns every partition of a same-name recreated + * topic, no partition lands in newly_added, no PARTITION_JOIN -> + * share_session_toppar_add fires for the NEW rktp, and ShareFetch + * never asks for the NEW generation. + * + * To deterministically expose the bug we need the consumer's heartbeat + * to deliver a direct OLD-id -> NEW-id target transition with no + * intermediate empty assignment. The mock cluster's heartbeat handler + * normally only recalculates assignments on member events (join, + * leave, subscription change) -- not on topic delete/recreate -- so a + * naive delete + recreate would leave the mock emitting the stale + * OLD-id assignment forever, never reaching the reconciler bug + * condition. Instead, the test uses + * rd_kafka_mock_sharegroup_target_assignment to manually inject the + * NEW-id assignment after the recreate. The mock resolves the + * partition list's topic name to the NEW topic_id at assignment-set + * time, so the consumer's next heartbeat carries + * (NEW-id, partition) tuples while its current assignment still holds + * the (OLD-id, partition) tuples from PHASE 1 -- exactly the input + * that triggers the (topic_name, partition) hash collision. + * + * Pre-fix: PHASE 2 times out with zero NEW records consumed. + * Post-fix: PHASE 2 consumes all NEW records. + */ +static void do_test_recreate_same_name_partition_collision(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_kafka_conf_t *producer_conf; + rd_kafka_t *producer; + const char *topic = "0184-recreate-collision"; + const char *group = "0184-share-recreate-collision"; + const int n_partitions = 3; + const int n_msgs_per_part = 10; + const int total_per_phase = n_partitions * n_msgs_per_part; + rd_kafka_messages_t *batch = NULL; + int old_got = 0; + int new_got = 0; + rd_ts_t deadline; + int32_t p; + char **member_ids; + size_t member_cnt; + size_t mi; + rd_kafka_resp_err_t err; + rd_kafka_topic_partition_list_t *new_assignment; + rd_kafka_topic_partition_list_t *member_assignments[1]; + const char *member_id_arr[1]; + + SUB_TEST_QUICK(); + + mcluster = test_mock_cluster_new(1, &bootstraps); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + /* Short heartbeat keeps the post-injection latency tight; the + * consumer's next heartbeat after the manual assignment-set + * delivers the NEW-id target within ~1s. */ + rd_kafka_mock_sharegroup_set_heartbeat_interval(mcluster, 1000); + + /* Persistent producer for the whole test. The mock cluster + * destroys every share-group session on the broker when any + * client connection closes (see + * rd_kafka_mock_sharegrps_node_connection_closed in + * rdkafka_mock_sharegrp.c), so spinning up a fresh producer + * instance per produce call would invalidate the consumer's + * session as collateral damage and break PHASE 2's fetch. + * Keeping one producer alive end-to-end avoids that. */ + test_conf_init(&producer_conf, NULL, 0); + test_conf_set(producer_conf, "bootstrap.servers", bootstraps); + rd_kafka_conf_set_dr_msg_cb(producer_conf, test_dr_msg_cb); + producer = test_create_handle(RD_KAFKA_PRODUCER, producer_conf); + TEST_ASSERT(producer != NULL, "producer create failed"); + + /* Initial topic generation. */ + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, n_partitions, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "mock_topic_create (initial) failed"); + + /* Produce n_msgs_per_part records to every partition of the + * initial generation so PHASE 1 has something to drain. */ + for (p = 0; p < n_partitions; p++) + test_produce_msgs2(producer, topic, 0 /*testid*/, p, + 0 /*msg_base*/, n_msgs_per_part, + NULL /*payload*/, 0); + + /* Long topic.metadata.refresh.interval.ms forces the heartbeat + * to be the channel that carries topic_id changes, which is + * what the reconciler bug rides on. */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group); + test_conf_set(conf, "share.acknowledgement.mode", "explicit"); + test_conf_set(conf, "topic.metadata.refresh.interval.ms", "300000"); + rkshare = rd_kafka_share_consumer_new(conf, NULL, 0); + TEST_ASSERT(rkshare != NULL, "share_consumer_new failed"); + + test_share_consumer_subscribe_multi(rkshare, 1, topic); + + /* ---- PHASE 1: drain the OLD generation ---- + * Single consumer in the share group => it owns every + * partition in this generation. Explicit-ack each record so + * the next consume_batch can proceed. */ + TEST_SAY("PHASE 1: draining %d OLD-generation record(s)\n", + total_per_phase); + deadline = test_clock() + (rd_ts_t)30 * 1000 * 1000; + while (old_got < total_per_phase && test_clock() < deadline) { + rd_kafka_error_t *error; + size_t cnt; + size_t i; + + error = rd_kafka_share_poll(rkshare, 500, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + cnt = rd_kafka_messages_count(batch); + for (i = 0; i < cnt; i++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, i); + if (!m->err) { + old_got++; + rd_kafka_share_acknowledge(rkshare, m); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(old_got == total_per_phase, + "PHASE 1: expected %d OLD records, got %d", total_per_phase, + old_got); + + /* Capture the consumer's member id so we can pin a target + * assignment for it after the recreate. */ + err = rd_kafka_mock_sharegroup_get_member_ids(mcluster, group, + &member_ids, &member_cnt); + TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR, + "mock_sharegroup_get_member_ids failed: %s", + rd_kafka_err2str(err)); + TEST_ASSERT(member_cnt == 1, + "expected exactly 1 member in share group, got %zu", + member_cnt); + + /* ---- Delete + recreate ---- + * Mock generates a fresh topic_id for the recreated topic. The + * existing share-group member's stored assignment still + * references the destroyed OLD topic; we override it below. */ + TEST_SAY("Recreating topic same name, same partition count\n"); + TEST_ASSERT(rd_kafka_mock_topic_delete(mcluster, topic) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "mock_topic_delete failed"); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, n_partitions, + 1) == RD_KAFKA_RESP_ERR_NO_ERROR, + "mock_topic_create (recreate) failed"); + + /* Pin the member's target assignment to the recreated topic. + * mock_sharegroup_target_assignment resolves the topic name + * to the current mtopic's id at set-time, so the partition + * list ends up carrying the NEW topic_id. The mock bumps the + * member epoch on set so the consumer's next heartbeat + * receives this assignment. From the consumer's point of + * view: current rkcg_group_assignment still has the OLD-id + * partitions from PHASE 1; the next HB target carries the + * NEW-id partitions. That is the precise input that triggers + * the reconciler's (topic_name, partition) hash collision. */ + new_assignment = rd_kafka_topic_partition_list_new(n_partitions); + for (p = 0; p < n_partitions; p++) + rd_kafka_topic_partition_list_add(new_assignment, topic, p); + member_id_arr[0] = member_ids[0]; + member_assignments[0] = new_assignment; + rd_kafka_mock_sharegroup_target_assignment( + mcluster, group, member_id_arr, member_assignments, 1); + rd_kafka_topic_partition_list_destroy(new_assignment); + + /* Produce records to every partition of the recreated topic. + * Pre-fix these never reach the consumer. */ + for (p = 0; p < n_partitions; p++) + test_produce_msgs2(producer, topic, 0 /*testid*/, p, + 0 /*msg_base*/, n_msgs_per_part, + NULL /*payload*/, 0); + + /* ---- PHASE 2: drain the NEW generation ---- + * Generous deadline so the heartbeat has time to deliver the + * NEW target, metadata to resolve for the new topic_id, and + * the share session to start fetching when the fix is in + * place. Pre-fix the reconciler diff collapses + * (name, partition) so the NEW rktps are never added to the + * share session; this loop hits the deadline at zero. */ + TEST_SAY("PHASE 2: draining %d NEW-generation record(s)\n", + total_per_phase); + deadline = test_clock() + (rd_ts_t)10 * 1000 * 1000; + while (new_got < total_per_phase && test_clock() < deadline) { + rd_kafka_error_t *error; + size_t cnt; + size_t i; + + error = rd_kafka_share_poll(rkshare, 500, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + cnt = rd_kafka_messages_count(batch); + for (i = 0; i < cnt; i++) { + rd_kafka_message_t *m = rd_kafka_messages_get(batch, i); + if (!m->err) { + new_got++; + rd_kafka_share_acknowledge(rkshare, m); + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + + TEST_ASSERT(new_got == total_per_phase, + "PHASE 2: expected %d NEW records after same-name " + "recreate, got %d. Cooperative reconciler likely " + "suppressed the NEW rktp adds because (name, partition) " + "collided with OLD-id entries in rkcg_group_assignment.", + total_per_phase, new_got); + + for (mi = 0; mi < member_cnt; mi++) + rd_free(member_ids[mi]); + rd_free(member_ids); + + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + rd_kafka_destroy(producer); + test_mock_cluster_destroy(mcluster); + SUB_TEST_PASS(); +} + +int main_0184_share_consumer_topic_recreate_local(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + test_timeout_set(180); + test_recreate_during_close(); + do_test_recreate_same_name_partition_collision(); + return 0; +} \ No newline at end of file diff --git a/lib/librdkafka-2.15.0/tests/0185-share_consumer_max_poll_interval.c b/lib/librdkafka-2.15.0/tests/0185-share_consumer_max_poll_interval.c new file mode 100644 index 00000000000..79fe3e2eda2 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0185-share_consumer_max_poll_interval.c @@ -0,0 +1,557 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "rdkafka.h" + +/** + * @name Share consumer max.poll.interval.ms enforcement (KIP-932). + * + * Recreates, for the share consumer, the applicable cases from + * 0089-max_poll_interval.c and 0091-max_poll_interval_timeout.c: + * + * - A share consumer that stops calling rd_kafka_share_poll() for + * longer than max.poll.interval.ms is considered failed, leaves the share + * group, and surfaces RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED — even if it + * never polled once. + * - The interval is reset only by rd_kafka_share_poll(); other calls + * (rd_kafka_poll(), polling the log queue) do not reset it. + * - A blocking rd_kafka_share_poll() that waits longer than + * max.poll.interval.ms is not counted against the interval. + * - Steady consumption within the interval never triggers the timeout. + * - After a timeout the consumer rejoins on the next batch poll. + * - A second consumer keeps consuming while the first is evicted. + * + */ + +/* Longer interval for the real-broker tests (matches 0089/0091), giving the + * group join/assignment enough headroom not to falsely trip. */ +#define REAL_MAX_POLL_INTERVAL_MS 10000 + + +/** + * @brief Create a share consumer with the given max.poll.interval.ms. + * + * @param bootstraps Mock bootstrap servers, or NULL to use the test.conf + * (real broker) default. + */ +static rd_kafka_share_t *create_share_consumer(const char *bootstraps, + const char *group_id, + int max_poll_interval_ms) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + char tmp[32]; + + test_conf_init(&conf, NULL, 0); + if (bootstraps) + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + rd_snprintf(tmp, sizeof(tmp), "%d", max_poll_interval_ms); + test_conf_set(conf, "max.poll.interval.ms", tmp); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer: %s", + errstr); + + return rkshare; +} + + +/** + * @brief Consume one batch. + * + * @param max_poll_exceeded If non-NULL, set to rd_true if the batch returned + * RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED. + * @returns the number of valid (non-error) messages received. + */ +static int share_consume_once(rd_kafka_share_t *rkshare, + int timeout_ms, + rd_bool_t *max_poll_exceeded) { + rd_kafka_messages_t *batch = NULL; + size_t rcvd = 0; + size_t i; + int valid = 0; + rd_kafka_error_t *error; + + if (max_poll_exceeded) + *max_poll_exceeded = rd_false; + + error = rd_kafka_share_poll(rkshare, timeout_ms, &batch); + if (error) { + if (max_poll_exceeded && + rd_kafka_error_code(error) == + RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED) + *max_poll_exceeded = rd_true; + rd_kafka_error_destroy(error); + return 0; + } + + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = rd_kafka_messages_get(batch, i); + if (!msg->err) + valid++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + + return valid; +} + + +/** + * @brief Consume batches until at least one message is received, establishing + * share-group membership and a steady assignment. Fails if no message + * arrives in time. + */ +static void establish_membership(rd_kafka_share_t *rkshare) { + int attempts; + + for (attempts = 0; attempts < 30; attempts++) { + if (share_consume_once(rkshare, 1000, NULL) > 0) { + TEST_SAY( + "Membership established (consumed a message)\n"); + return; + } + } + + TEST_FAIL("Failed to consume any message to establish membership"); +} + + +/** + * @brief Poll batches until RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED is observed + * or @p timeout_ms elapses. + * + * @returns rd_true if the error was observed. + */ +static rd_bool_t wait_max_poll_exceeded(rd_kafka_share_t *rkshare, + int timeout_ms) { + int64_t deadline = test_clock() + (int64_t)timeout_ms * 1000; + + while (test_clock() < deadline) { + rd_bool_t exceeded = rd_false; + share_consume_once(rkshare, 500, &exceeded); + if (exceeded) + return rd_true; + } + + return rd_false; +} + + +/** + * @brief Real-broker setup: create a single-partition topic, produce @p msgcnt + * messages, set the share group's offset reset to earliest, then create + * a subscribed share consumer with the given max.poll.interval.ms. + * + * @param topic Output buffer for the generated topic name. + */ +static rd_kafka_share_t * +real_setup(char *topic, size_t topic_sz, int msgcnt, int max_poll_interval_ms) { + char group[64]; + uint64_t testid = test_id_generate(); + rd_kafka_share_t *c; + + rd_snprintf(topic, topic_sz, "%s", + test_mk_topic_name("0185_share_max_poll", 1)); + test_str_id_generate(group, sizeof(group)); + + test_create_topic_wait_exists(NULL, topic, 1, -1, 5000); + if (msgcnt > 0) + test_produce_msgs_easy(topic, testid, 0, msgcnt); + test_share_set_auto_offset_reset(group, "earliest"); + + c = create_share_consumer(NULL, group, max_poll_interval_ms); + test_share_consumer_subscribe_multi(c, 1, topic); + + return c; +} + + +/* + * =========================================================================== + * Real broker tests + * =========================================================================== + */ + +/** + * @brief Long "processing" (no polling) past max.poll.interval.ms must evict + * the consumer with MAX_POLL_EXCEEDED, after which it can rejoin. + * + * Mirrors 0089 do_test() + do_test_rejoin_after_interval_expire(). + */ +static void do_test_long_processing(void) { + char topic[128]; + rd_kafka_share_t *c; + int consumed = 0; + int attempts; + + SUB_TEST(); + + c = real_setup(topic, sizeof(topic), 20, REAL_MAX_POLL_INTERVAL_MS); + establish_membership(c); + + TEST_SAY( + "Simulating long processing: sleeping %ds " + "(> max.poll.interval.ms=%dms)\n", + (REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000, + REAL_MAX_POLL_INTERVAL_MS); + rd_sleep((REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000); + + TEST_ASSERT(wait_max_poll_exceeded(c, 10000), + "Expected MAX_POLL_EXCEEDED after long processing"); + + /* The records consumed before the stall are implicitly acknowledged + * while detecting the timeout (implicit ack is sent at the start of + * the consume_batch() that returns the error), so the share-partition + * start offset has advanced past them. Produce fresh records and + * verify the consumer rejoins and resumes consuming. */ + test_produce_msgs_easy(topic, test_id_generate(), 0, 20); + + for (attempts = 0; attempts < 30 && consumed == 0; attempts++) + consumed += share_consume_once(c, 1000, NULL); + + TEST_ASSERT(consumed > 0, + "Consumer did not rejoin after max.poll.interval.ms"); + + test_share_consumer_close(c); + test_share_destroy(c); + + SUB_TEST_PASS(); +} + + +/** + * @brief A share consumer that subscribes but NEVER calls consume_batch must + * still be evicted with MAX_POLL_EXCEEDED once the interval elapses. + * + * Mirrors 0091 do_test_no_poll(). + */ +static void do_test_no_poll(void) { + char topic[128]; + rd_kafka_share_t *c; + + SUB_TEST(); + + c = real_setup(topic, sizeof(topic), 20, REAL_MAX_POLL_INTERVAL_MS); + + /* Do NOT consume at all: the background heartbeat joins the group and + * receives an assignment, which starts the timer; rk_ts_last_poll + * stays at its initial value, so the interval elapses with no poll. */ + TEST_SAY("Subscribed but never polling; sleeping %ds (> %dms)\n", + (REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000, + REAL_MAX_POLL_INTERVAL_MS); + rd_sleep((REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000); + + TEST_ASSERT( + wait_max_poll_exceeded(c, 10000), + "Expected MAX_POLL_EXCEEDED even though consume_batch() was " + "never called before the stall"); + + test_share_consumer_close(c); + test_share_destroy(c); + + SUB_TEST_PASS(); +} + + +/** + * @brief Steady consumption with per-batch processing below + * max.poll.interval.ms must not evict the consumer. + * + * Mirrors 0091 do_test_with_subscribe(). + */ +static void do_test_steady_consume(void) { + char topic[128]; + rd_kafka_share_t *c; + const int msgcnt = 30; + int consumed = 0; + int64_t deadline; + + SUB_TEST(); + + c = real_setup(topic, sizeof(topic), msgcnt, REAL_MAX_POLL_INTERVAL_MS); + + /* Consume over a span longer than max.poll.interval.ms, with a short + * "processing" sleep between polls (all under the interval). */ + deadline = + test_clock() + (int64_t)(REAL_MAX_POLL_INTERVAL_MS + 5000) * 1000; + while (consumed < msgcnt && test_clock() < deadline) { + rd_bool_t exceeded = rd_false; + consumed += share_consume_once(c, 2000, &exceeded); + TEST_ASSERT(!exceeded, + "Steady consumption must not trip " + "max.poll.interval.ms (consumed %d/%d)", + consumed, msgcnt); + rd_sleep(1); + } + + TEST_ASSERT(consumed >= msgcnt, "Expected %d messages, consumed %d", + msgcnt, consumed); + + test_share_consumer_close(c); + test_share_destroy(c); + + SUB_TEST_PASS(); +} + + +/** + * @brief Two share consumers in the same group: the one that stops polling for + * longer than max.poll.interval.ms is evicted with MAX_POLL_EXCEEDED, + * while the other keeps consuming. + * + * Mirrors 0089 do_test() (two consumers, one stalls). Note: unlike a classic + * consumer group a share group has no exclusive partition ownership, so the + * second consumer consumes concurrently and is not gated on the first's + * eviction — that timing assertion from 0089 does not apply here. + */ +static void do_test_two_consumers(void) { + const char *topic = test_mk_topic_name("0185_share_max_poll", 1); + char group[64]; + uint64_t testid; + rd_kafka_share_t *c[2]; + int other_consumed = 0; + int i; + int64_t deadline; + + SUB_TEST(); + + testid = test_id_generate(); + test_str_id_generate(group, sizeof(group)); + + test_create_topic_wait_exists(NULL, topic, 1, -1, 5000); + test_produce_msgs_easy(topic, testid, 0, 100); + + for (i = 0; i < 2; i++) + c[i] = create_share_consumer(NULL, group, + REAL_MAX_POLL_INTERVAL_MS); + + test_share_set_auto_offset_reset(group, "earliest"); + + for (i = 0; i < 2; i++) + test_share_consumer_subscribe_multi(c[i], 1, topic); + + /* c[0] never polls: it joins the group and is assigned via heartbeat + * (which starts its max.poll.interval.ms timer) but immediately stalls. + * Since it never fetches, it never acquires records, so c[1] can + * consume freely. c[1] keeps polling for longer than the interval, + * staying in the group and consuming. */ + TEST_SAY("c[0] stalls (never polls); c[1] keeps consuming for %ds\n", + (REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000); + deadline = + test_clock() + (int64_t)(REAL_MAX_POLL_INTERVAL_MS + 3000) * 1000; + while (test_clock() < deadline) + other_consumed += share_consume_once(c[1], 1000, NULL); + + TEST_ASSERT(other_consumed > 0, + "c[1] should have kept consuming while c[0] stalled"); + + /* c[0] must surface MAX_POLL_EXCEEDED on its next poll (the error was + * enqueued by the timer while it was stalled). */ + TEST_ASSERT(wait_max_poll_exceeded(c[0], 10000), + "c[0] (stalled) expected MAX_POLL_EXCEEDED"); + + for (i = 0; i < 2; i++) { + test_share_consumer_close(c[i]); + test_share_destroy(c[i]); + } + + SUB_TEST_PASS(); +} + + +/** + * @brief While "processing", polling only the log queue must NOT reset the + * share consumer's max.poll.interval.ms timer. + * + * Mirrors 0089 do_test_with_log_queue(). + */ +static void do_test_log_queue_no_reset(void) { + const char *topic = test_mk_topic_name("0185_share_max_poll", 1); + char group[64]; + uint64_t testid; + rd_kafka_conf_t *conf; + rd_kafka_share_t *c; + rd_kafka_t *rk; + rd_kafka_queue_t *logq; + char errstr[512]; + char tmp[32]; + int64_t deadline; + + SUB_TEST(); + + testid = test_id_generate(); + test_str_id_generate(group, sizeof(group)); + + test_create_topic_wait_exists(NULL, topic, 1, -1, 5000); + test_produce_msgs_easy(topic, testid, 0, 20); + test_share_set_auto_offset_reset(group, "earliest"); + + /* Build a consumer with log.queue=true so logs are routed to a + * dedicated queue that we can poll during "processing". */ + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "group.id", group); + rd_snprintf(tmp, sizeof(tmp), "%d", REAL_MAX_POLL_INTERVAL_MS); + test_conf_set(conf, "max.poll.interval.ms", tmp); + test_conf_set(conf, "log.queue", "true"); + c = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(c, "Failed to create share consumer: %s", errstr); + + rk = test_share_consumer_get_rk(c); + logq = rd_kafka_queue_new(rk); + TEST_CALL__(rd_kafka_set_log_queue(rk, logq)); + + test_share_consumer_subscribe_multi(c, 1, topic); + establish_membership(c); + + TEST_SAY("Polling only the log queue for %ds (> %dms)\n", + (REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000, + REAL_MAX_POLL_INTERVAL_MS); + deadline = + test_clock() + (int64_t)(REAL_MAX_POLL_INTERVAL_MS + 3000) * 1000; + while (test_clock() < deadline) + rd_kafka_event_destroy(rd_kafka_queue_poll(logq, 200)); + + TEST_ASSERT( + wait_max_poll_exceeded(c, 10000), + "Expected MAX_POLL_EXCEEDED: polling the log queue must not " + "reset the share consumer max.poll.interval.ms timer"); + + test_share_consumer_close(c); + test_share_destroy(c); + rd_kafka_queue_destroy(logq); + + SUB_TEST_PASS(); +} + + +/** + * @brief rd_kafka_poll() on the underlying handle must NOT reset the share + * consumer's max.poll.interval.ms timer; only + * rd_kafka_share_poll() does. + * + * Inverse of 0089 do_test_max_poll_reset_with_consumer_cb(): the regular + * consumer's poll resets the timer, the share consumer's does not. + */ +static void do_test_rd_kafka_poll_does_not_reset(void) { + char topic[128]; + rd_kafka_share_t *c; + rd_kafka_t *rk; + int64_t deadline; + + SUB_TEST(); + + c = real_setup(topic, sizeof(topic), 20, REAL_MAX_POLL_INTERVAL_MS); + establish_membership(c); + + rk = test_share_consumer_get_rk(c); + + TEST_SAY("Calling only rd_kafka_poll() for %ds (> %dms)\n", + (REAL_MAX_POLL_INTERVAL_MS + 3000) / 1000, + REAL_MAX_POLL_INTERVAL_MS); + deadline = + test_clock() + (int64_t)(REAL_MAX_POLL_INTERVAL_MS + 3000) * 1000; + while (test_clock() < deadline) + rd_kafka_poll(rk, 200); + + TEST_ASSERT( + wait_max_poll_exceeded(c, 10000), + "Expected MAX_POLL_EXCEEDED: rd_kafka_poll() must not reset " + "the share consumer max.poll.interval.ms timer"); + + test_share_consumer_close(c); + test_share_destroy(c); + + SUB_TEST_PASS(); +} + + +/** + * @brief A single blocking rd_kafka_share_poll() that waits longer + * than max.poll.interval.ms must NOT trip the timeout (the wait is time + * spent inside librdkafka, not application processing time). + */ +static void do_test_blocking_poll_no_false_trip(void) { + char topic[128]; + rd_kafka_share_t *c; + rd_bool_t exceeded = rd_false; + int consumed = 0; + int attempts; + + SUB_TEST(); + + c = real_setup(topic, sizeof(topic), 5, REAL_MAX_POLL_INTERVAL_MS); + establish_membership(c); + + /* Drain remaining produced messages so the next poll blocks. */ + while (share_consume_once(c, 500, NULL) > 0) + ; + + TEST_SAY("Blocking in a single consume_batch() for %ds (> %dms)\n", + (REAL_MAX_POLL_INTERVAL_MS + 2000) / 1000, + REAL_MAX_POLL_INTERVAL_MS); + share_consume_once(c, REAL_MAX_POLL_INTERVAL_MS + 2000, &exceeded); + + TEST_ASSERT(!exceeded, + "A blocking consume_batch() must not trip " + "max.poll.interval.ms"); + + test_produce_msgs_easy(topic, test_id_generate(), 0, 5); + for (attempts = 0; attempts < 30 && consumed == 0; attempts++) + consumed += share_consume_once(c, 1000, &exceeded); + + TEST_ASSERT(!exceeded, + "Consumer should remain in the group after a long " + "blocking poll"); + TEST_ASSERT(consumed > 0, + "Consumer should still consume messages after a long " + "blocking poll"); + + test_share_consumer_close(c); + test_share_destroy(c); + + SUB_TEST_PASS(); +} + + +int main_0185_share_consumer_max_poll_interval(int argc, char **argv) { + test_timeout_set(300); + + do_test_long_processing(); + do_test_no_poll(); + do_test_steady_consume(); + do_test_two_consumers(); + do_test_log_queue_no_reset(); + do_test_rd_kafka_poll_does_not_reset(); + do_test_blocking_poll_no_false_trip(); + + return 0; +} diff --git a/lib/librdkafka-2.15.0/tests/0186-share_consumer_fatal_error.c b/lib/librdkafka-2.15.0/tests/0186-share_consumer_fatal_error.c new file mode 100644 index 00000000000..2c9a18dbe4b --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0186-share_consumer_fatal_error.c @@ -0,0 +1,360 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include "../src/rdkafka_proto.h" + +#define CONSUME_ARRAY 1024 + +static rd_bool_t is_share_heartbeat_request(rd_kafka_mock_request_t *request, + void *opaque) { + return rd_kafka_mock_request_api_key(request) == + RD_KAFKAP_ShareGroupHeartbeat; +} + +/** + * @brief Create a share consumer connected to the mock cluster. + * + * @param ack_mode "implicit" or "explicit" (NULL for the default). + * @param cb_state Opaque ack-commit-callback state (NULL to skip the cb). + */ +static rd_kafka_share_t *create_share_consumer(const char *bootstraps, + const char *group_id, + const char *ack_mode, + test_ack_cb_state_t *cb_state) { + rd_kafka_conf_t *conf; + rd_kafka_share_t *rkshare; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + test_conf_set(conf, "bootstrap.servers", bootstraps); + test_conf_set(conf, "group.id", group_id); + if (ack_mode) + test_conf_set(conf, "share.acknowledgement.mode", ack_mode); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare != NULL, "Failed to create share consumer: %s", + errstr); + + if (cb_state) { + rd_kafka_error_t *cb_err = + rd_kafka_share_set_acknowledgement_commit_cb( + rkshare, test_share_ack_cb, cb_state); + TEST_ASSERT(cb_err == NULL, + "Failed to set ack commit callback: %s", + rd_kafka_error_string(cb_err)); + } + + return rkshare; +} + +/** + * @brief No more records after the first fatal error, even when records + * are available on the broker. + * + * 1. Create a topic. Produce 10 messages. + * 2. Consume them via consume_batch (implicit ack). + * 3. Inject a fatal error (GROUP_AUTHORIZATION_FAILED) on the next + * ShareGroupHeartbeat and poll consume_batch until the fatal error + * surfaces. No records are available, so every call returns zero. + * 4. Produce 10 more messages: these are now available on the broker. + * 5. Every consume_batch call must still return zero records, since the + * consumer is in a fatal state. + * 6. Close and destroy the consumer. + */ +static void do_test_no_records_after_fatal_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "sg-0186-no-records-after-fatal"; + const rd_kafka_resp_err_t fatal_err = + RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED; + const int msgcnt = 10; + int consumed; + int attempts; + rd_bool_t got_fatal = rd_false; + int i; + + SUB_TEST_QUICK(); + + /* 1. Setup mock cluster and a topic, produce 10 messages. */ + mcluster = test_mock_cluster_new(1, &bootstraps); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + test_produce_msgs_easy_v(topic, 0, 0, 0, msgcnt, 16, + "bootstrap.servers", bootstraps, NULL); + + /* Create the share consumer (implicit ack) and subscribe. */ + rkshare = create_share_consumer(bootstraps, group, "implicit", NULL); + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + rd_kafka_share_subscribe(rkshare, subscription); + rd_kafka_topic_partition_list_destroy(subscription); + + /* 2. Consume all messages (implicit ack). */ + consumed = test_share_consume_msgs(rkshare, msgcnt, 30, 3000, NULL, 0); + TEST_ASSERT(consumed == msgcnt, "expected %d consumed, got %d", msgcnt, + consumed); + + /* 3. Inject the fatal error on the next ShareGroupHeartbeat and poll + * until it surfaces via consume_batch. No records are available + * yet, so every call must return zero records. */ + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + fatal_err, 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push fatal error on ShareGroupHeartbeat"); + + attempts = 0; + while (attempts++ < 50) { + size_t rcvd; + error = rd_kafka_share_poll(rkshare, 1000, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "no records expected while waiting for the fatal " + "error, got %d", + (int)rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + if (error) { + TEST_ASSERT(rd_kafka_error_is_fatal(error), + "expected a fatal error, got non-fatal %s", + rd_kafka_error_name(error)); + TEST_ASSERT(rd_kafka_error_code(error) == fatal_err, + "expected fatal %s, got %s", + rd_kafka_err2name(fatal_err), + rd_kafka_error_name(error)); + rd_kafka_error_destroy(error); + got_fatal = rd_true; + break; + } + } + TEST_ASSERT(got_fatal, + "expected a fatal error to surface within timeout"); + + /* 4. Produce more messages: they are now available on the broker. */ + test_produce_msgs_easy_v(topic, 0, 0, 0, msgcnt, 16, + "bootstrap.servers", bootstraps, NULL); + + /* 5. Every subsequent consume_batch call must keep returning zero + * records, even though new messages are available. */ + for (i = 0; i < 10; i++) { + size_t rcvd; + error = rd_kafka_share_poll(rkshare, 500, &batch); + rcvd = rd_kafka_messages_count(batch); + TEST_ASSERT(rcvd == 0, + "expected 0 records on post-fatal call %d, got %d", + i, (int)rcvd); + rd_kafka_messages_destroy(batch); + batch = NULL; + if (error) + rd_kafka_error_destroy(error); + } + + /* 6. Close and destroy. */ + test_share_consumer_close(rkshare); + test_share_destroy(rkshare); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +/** + * @brief close() flushes pending acks but does NOT send a share-group + * leave heartbeat when a fatal error was raised naturally. + * + * When a fatal error arises from the ShareGroupHeartbeat path the member + * is no longer in a state to leave the group, so close() must not send a + * leaving ShareGroupHeartbeat. It must still flush the pending + * acknowledgements (so the ack commit callback fires without error). + * + * 1. Create a 1-partition topic and produce 10 messages. + * 2. Consume at least one batch (explicit ack), acknowledging every + * record so the acks are pending (not yet sent to the broker). + * 3. Inject GROUP_AUTHORIZATION_FAILED on the next ShareGroupHeartbeat + * and wait for a heartbeat so the fatal error is raised naturally. + * 4. Poll consume_batch until the fatal error surfaces. + * 5. Start request tracking, then call close(). + * 6. Assert NO ShareGroupHeartbeat (leave) request is sent during close. + * 7. Destroy, and verify the ack commit callback fired without error. + */ +static void do_test_close_flushes_acks_after_fatal_error(void) { + rd_kafka_mock_cluster_t *mcluster; + const char *bootstraps; + rd_kafka_share_t *rkshare; + rd_kafka_topic_partition_list_t *subscription; + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error; + const char *topic = test_mk_topic_name(__FUNCTION__, 0); + const char *group = "sg-0186-close-no-leave-after-fatal"; + const rd_kafka_resp_err_t fatal_err = + RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED; + const int msgcnt = 10; + test_ack_cb_state_t cb_state = {0}; + int heartbeats_after_close; + int consumed = 0; + int attempts = 0; + rd_bool_t got_fatal = rd_false; + + SUB_TEST_QUICK(); + + /* 1. Setup mock cluster and a topic, produce 10 messages. */ + mcluster = test_mock_cluster_new(1, &bootstraps); + TEST_ASSERT(rd_kafka_mock_topic_create(mcluster, topic, 1, 1) == + RD_KAFKA_RESP_ERR_NO_ERROR, + "create topic"); + rd_kafka_mock_sharegroup_set_auto_offset_reset(mcluster, 1); + + test_produce_msgs_easy_v(topic, 0, 0, 0, msgcnt, 16, + "bootstrap.servers", bootstraps, NULL); + + /* Create an explicit-ack share consumer with a commit callback. */ + rkshare = + create_share_consumer(bootstraps, group, "explicit", &cb_state); + subscription = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subscription, topic, + RD_KAFKA_PARTITION_UA); + TEST_CALL_ERR__(rd_kafka_share_subscribe(rkshare, subscription)); + rd_kafka_topic_partition_list_destroy(subscription); + + /* 2. Consume at least one batch and acknowledge every record. + * Break out as soon as we have received some records. */ + while (consumed == 0 && attempts++ < 30) { + size_t rcvd; + size_t j; + error = rd_kafka_share_poll(rkshare, 3000, &batch); + if (error) { + rd_kafka_error_destroy(error); + rd_kafka_messages_destroy(batch); + batch = NULL; + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (j = 0; j < rcvd; j++) { + rd_kafka_message_t *rkm = + rd_kafka_messages_get(batch, j); + if (!rkm->err) { + TEST_CALL_ERR__( + rd_kafka_share_acknowledge(rkshare, rkm)); + consumed++; + } + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_ASSERT(consumed > 0, "expected to consume and ack > 0 records"); + TEST_SAY("Consumed and acknowledged %d records\n", consumed); + + /* 3. Inject the fatal error on the next ShareGroupHeartbeat so the + * fatal error gets raised naturally. */ + TEST_ASSERT(rd_kafka_mock_broker_push_request_error_rtts( + mcluster, 1, RD_KAFKAP_ShareGroupHeartbeat, 1, + fatal_err, 0) == RD_KAFKA_RESP_ERR_NO_ERROR, + "push fatal error on ShareGroupHeartbeat"); + + /* 4. Poll consume_batch until the fatal error surfaces. The fatal + * error is delivered only after a heartbeat has hit the injected + * error and the cgrp has processed it (revoking the assignment + * and resetting the generation id), so this loop also serves as + * the synchronization point for that handling. */ + attempts = 0; + while (attempts++ < 50) { + error = rd_kafka_share_poll(rkshare, 1000, &batch); + rd_kafka_messages_destroy(batch); + batch = NULL; + if (error) { + TEST_ASSERT(rd_kafka_error_is_fatal(error), + "expected a fatal error, got non-fatal %s", + rd_kafka_error_name(error)); + rd_kafka_error_destroy(error); + got_fatal = rd_true; + break; + } + } + TEST_ASSERT(got_fatal, + "expected a fatal error to surface within timeout"); + + /* 5. Reset the tracked-request baseline (start_request_tracking + * clears the list), then close(). */ + rd_kafka_mock_start_request_tracking(mcluster); + + error = rd_kafka_share_consumer_close(rkshare); + if (error) + rd_kafka_error_destroy(error); + + /* 6. No leaving ShareGroupHeartbeat must be sent during close. The + * natural fatal handling already revoked the assignment and reset + * the generation id (HAS_JOINED is false), so the group-leave path + * is skipped. (The share session is still closed, but via a + * ShareAcknowledge with epoch=-1, not a ShareGroupHeartbeat.) */ + heartbeats_after_close = (int)test_mock_get_matching_request_cnt( + mcluster, is_share_heartbeat_request, NULL); + + rd_kafka_mock_stop_request_tracking(mcluster); + + /* 7. Destroy and verify the ack commit callback fired without error, + * confirming the pending acks were flushed during close(). */ + test_share_destroy(rkshare); + + TEST_ASSERT(heartbeats_after_close == 0, + "expected no ShareGroupHeartbeat during close, got %d", + heartbeats_after_close); + TEST_ASSERT(cb_state.callback_cnt > 0, + "expected the ack commit callback to be invoked, got %d", + cb_state.callback_cnt); + rd_kafka_resp_err_t cb_first_err = + test_ack_cb_state_first_err(&cb_state); + TEST_ASSERT(cb_first_err == RD_KAFKA_RESP_ERR_NO_ERROR, + "expected ack callback with no error, got %s", + rd_kafka_err2name(cb_first_err)); + TEST_SAY("Ack callback invoked %d time(s), %zu offsets, err=%s\n", + cb_state.callback_cnt, cb_state.total_offsets, + rd_kafka_err2name(cb_first_err)); + + test_ack_cb_state_destroy(&cb_state); + test_mock_cluster_destroy(mcluster); + + SUB_TEST_PASS(); +} + +int main_0186_share_consumer_fatal_error(int argc, char **argv) { + TEST_SKIP_MOCK_CLUSTER(0); + + do_test_no_records_after_fatal_error(); + do_test_close_flushes_acks_after_fatal_error(); + + return 0; +} \ No newline at end of file diff --git a/lib/librdkafka-2.15.0/tests/0190-share_consumer_telemetry.c b/lib/librdkafka-2.15.0/tests/0190-share_consumer_telemetry.c new file mode 100644 index 00000000000..fee4cd70d2e --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/0190-share_consumer_telemetry.c @@ -0,0 +1,387 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2026, Confluent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include "rdkafka.h" + +#include +#include +#include + +#define TELEMETRY_TOPIC_DEFAULT "client-telemetry-metrics" +#define METRIC_PREFIX "org.apache.kafka." + +/* Expected share-consumer metric suffixes (without the org.apache.kafka. + * prefix). Mirrors RD_KAFKA_TELEMETRY_SHARE_CONSUMER_METRICS_INFO in + * src/rdkafka_telemetry_encode.h; hardcoded here so the test doesn't + * transitively include the internal header. */ +static const char *const EXPECTED_SHARE_CONSUMER_METRICS[] = { + "consumer.share.poll.idle.ratio.avg", + "consumer.share.time.between.poll.avg", + "consumer.share.time.between.poll.max", + "consumer.share.fetch.manager.fetch.total", + "consumer.share.fetch.manager.fetch.rate", + "consumer.share.fetch.manager.fetch.latency.avg", + "consumer.share.fetch.manager.fetch.latency.max", + "consumer.share.fetch.manager.acknowledgements.send.total", + "consumer.share.fetch.manager.acknowledgements.send.rate", + "consumer.share.fetch.manager.fetch.throttle.time.avg", + "consumer.share.fetch.manager.fetch.throttle.time.max", + "consumer.share.coordinator.heartbeat.total", + "consumer.share.coordinator.heartbeat.rate", + "consumer.share.fetch.manager.fetch.size.avg", + "consumer.share.fetch.manager.fetch.size.max", + "consumer.share.fetch.manager.bytes.consumed.total", + "consumer.share.fetch.manager.bytes.consumed.rate", +}; + +#define EXPECTED_SHARE_CONSUMER_METRICS_CNT \ + (sizeof(EXPECTED_SHARE_CONSUMER_METRICS) / \ + sizeof(EXPECTED_SHARE_CONSUMER_METRICS[0])) + +static const char *telemetry_topic(void) { + const char *env = getenv("TELEMETRY_TOPIC"); + return env && *env ? env : TELEMETRY_TOPIC_DEFAULT; +} + +/* Fixed-capacity set of unique metric names. */ +#define MAX_SEEN_METRICS 64 +#define MAX_METRIC_NAME_LEN 256 + +typedef struct { + char names[MAX_SEEN_METRICS][MAX_METRIC_NAME_LEN]; + int cnt; +} seen_metrics_t; + +static rd_bool_t seen_metrics_contains(const seen_metrics_t *seen, + const char *name) { + int i; + for (i = 0; i < seen->cnt; i++) { + if (strcmp(seen->names[i], name) == 0) + return rd_true; + } + return rd_false; +} + +static void seen_metrics_add(seen_metrics_t *seen, const char *name) { + if (seen_metrics_contains(seen, name)) + return; + if (seen->cnt >= MAX_SEEN_METRICS) { + TEST_FAIL( + "seen_metrics capacity (%d) exceeded; dropping '%s'\n", + MAX_SEEN_METRICS, name); + return; + } + rd_snprintf(seen->names[seen->cnt], MAX_METRIC_NAME_LEN, "%s", name); + seen->cnt++; +} + +/** + * @name End-to-end share-consumer telemetry integration test. + * + * Requires external infrastructure already brought up by the surrounding + * pipeline (packaging/tools/setup-share-telemetry-testing.sh): + * - broker with ClientOtlpMetricsReporter plugin loaded + * - OpenTelemetry Collector with Kafka exporter + * - topic "client-telemetry-metrics" (the OTel-exported telemetry sink) + * - client metrics subscription on "org.apache.kafka.consumer.share." prefix + */ + +/** + * @brief Scan a raw byte buffer for every + * "org.apache.kafka.consumer.share." substring and add each + * unique one to the names list. + * + * The OTLP protobuf encoding leaves metric names as plain UTF-8 — the + * field tag is binary but the string payload is uncompressed. + */ +static void +extract_share_metric_names(const void *data, size_t len, seen_metrics_t *seen) { + const char *p = (const char *)data; + const char *end = p + len; + const char prefix[] = METRIC_PREFIX "consumer.share."; + const size_t prefix_len = sizeof(prefix) - 1; + + while (p + prefix_len <= end) { + const char *match = NULL; + const char *q; + for (q = p; q + prefix_len <= end; q++) { + if (memcmp(q, prefix, prefix_len) == 0) { + match = q; + break; + } + } + if (!match) + break; + + const char *name_end = match + prefix_len; + while (name_end < end && + (isalnum((unsigned char)*name_end) || *name_end == '.' || + *name_end == '_')) { + name_end++; + } + + size_t name_len = (size_t)(name_end - match); + if (name_len > 0 && name_len < MAX_METRIC_NAME_LEN) { + char name[MAX_METRIC_NAME_LEN]; + memcpy(name, match, name_len); + name[name_len] = '\0'; + seen_metrics_add(seen, name); + } + + p = name_end; + } +} + +/** + * @brief Produce a batch of messages and drive a share consumer through + * the consume path so the telemetry sampling sites fire. + */ +static void produce_and_share_consume(const char *topic, const char *group_id) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + rd_kafka_resp_err_t err; + char errstr[512]; + int64_t deadline_us; + int consumed = 0; + + TEST_SAY("Producing %d messages to %s\n", 200, topic); + test_create_topic_wait_exists(NULL, topic, 1, -1, 30 * 1000); + test_produce_msgs_easy(topic, 0, RD_KAFKA_PARTITION_UA, 200); + + TEST_SAY( + "Starting share consumer (enable.metrics.push=true) for %d s\n", + 20); + + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "group.id", group_id); + test_conf_set(conf, "enable.metrics.push", "true"); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "share_consumer_new failed: %s", errstr); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + err = rd_kafka_share_subscribe(rkshare, subs); + rd_kafka_topic_partition_list_destroy(subs); + TEST_ASSERT(!err, "share_subscribe: %s", rd_kafka_err2str(err)); + + deadline_us = test_clock() + 20 * 1000000; + while (test_clock() < deadline_us) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *e; + size_t rcvd = 0; + size_t i; + + e = rd_kafka_share_poll(rkshare, 1000, &batch); + if (e) { + rd_kafka_error_destroy(e); + continue; + } + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = + rd_kafka_messages_get(batch, i); + if (msg && !msg->err) + consumed++; + } + rd_kafka_messages_destroy(batch); + batch = NULL; + } + TEST_SAY("Share-consumed %d messages\n", consumed); + + rd_kafka_share_consumer_close(rkshare); + rd_kafka_share_destroy(rkshare); +} + +/** + * @brief Open a regular consumer on the telemetry topic, drain it for + * up to the given number of seconds, returning a list of every + * distinct "consumer.share.*" metric name seen across all messages. + */ +static void +consume_telemetry_topic(const char *topic, int seconds, seen_metrics_t *seen) { + rd_kafka_t *rk; + rd_kafka_conf_t *conf; + rd_kafka_topic_partition_list_t *subs; + int64_t deadline_us; + int msgs_seen = 0; + + TEST_SAY("Consuming '%s' for up to %d s\n", topic, seconds); + + test_conf_init(&conf, NULL, 60); + test_conf_set(conf, "group.id", "0190-telemetry-verifier"); + test_conf_set(conf, "auto.offset.reset", "earliest"); + test_conf_set(conf, "enable.auto.commit", "false"); + + rk = test_create_handle(RD_KAFKA_CONSUMER, conf); + rd_kafka_poll_set_consumer(rk); + + subs = rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_add(subs, topic, RD_KAFKA_PARTITION_UA); + rd_kafka_subscribe(rk, subs); + rd_kafka_topic_partition_list_destroy(subs); + + deadline_us = test_clock() + (int64_t)seconds * 1000000; + while (test_clock() < deadline_us) { + rd_kafka_message_t *msg = rd_kafka_consumer_poll(rk, 1000); + if (!msg) + continue; + if (msg->err) { + rd_kafka_message_destroy(msg); + continue; + } + msgs_seen++; + extract_share_metric_names(msg->payload, msg->len, seen); + rd_kafka_message_destroy(msg); + } + + TEST_SAY( + "Read %d telemetry messages, observed %d distinct share metric " + "names\n", + msgs_seen, seen->cnt); + + rd_kafka_consumer_close(rk); + rd_kafka_destroy(rk); +} + +/** + * @brief Compare the seen-names list against EXPECTED_METRICS and + * TEST_ASSERT that every expected name appears with the full + * "org.apache.kafka." prefix. + */ +static void verify_metrics(const seen_metrics_t *seen) { + int missing = 0; + int expected_cnt = 0; + size_t i; + + TEST_SAY("Verifying expected share-consumer metric names:\n"); + for (i = 0; i < EXPECTED_SHARE_CONSUMER_METRICS_CNT; i++) { + char full_name[MAX_METRIC_NAME_LEN]; + const char *short_name = EXPECTED_SHARE_CONSUMER_METRICS[i]; + int found; + + rd_snprintf(full_name, sizeof(full_name), METRIC_PREFIX "%s", + short_name); + found = seen_metrics_contains(seen, full_name) ? 1 : 0; + expected_cnt++; + TEST_SAY(" %s %s\n", found ? "[OK]" : "[MISSING]", full_name); + if (!found) + missing++; + } + + TEST_ASSERT(missing == 0, + "%d of %d expected share-consumer metric name(s) " + "were not observed in topic '%s'", + missing, expected_cnt, telemetry_topic()); + + TEST_SAY("PASS: all %d expected share-consumer metric names observed\n", + expected_cnt); +} + + +/** + * @brief Return 1 if the given topic exists on the broker, 0 otherwise. + * + * Used to decide whether the surrounding telemetry pipeline is set up. + * If the OTel Collector is not running and producing into the topic, + * the topic itself won't exist and the test is skipped. CI pipelines + * that intend to exercise the full path are expected to verify infra + * health themselves before invoking this test. + */ +static int telemetry_infra_available(const char *topic) { + rd_kafka_t *rk; + rd_kafka_conf_t *conf; + const struct rd_kafka_metadata *md; + rd_kafka_resp_err_t err; + int found = 0; + int i; + + test_conf_init(&conf, NULL, 30); + rk = test_create_handle(RD_KAFKA_PRODUCER, conf); + + /* List ALL topics — does NOT trigger broker-side auto-create the + * way a per-topic metadata query would on clusters that have + * auto.create.topics.enable=true (e.g. trivup). */ + err = rd_kafka_metadata(rk, 1, NULL, &md, 10 * 1000); + if (err == RD_KAFKA_RESP_ERR_NO_ERROR) { + for (i = 0; i < md->topic_cnt; i++) { + if (strcmp(md->topics[i].topic, topic) == 0 && + md->topics[i].err == RD_KAFKA_RESP_ERR_NO_ERROR && + md->topics[i].partition_cnt > 0) { + found = 1; + break; + } + } + rd_kafka_metadata_destroy(md); + } + + rd_kafka_destroy(rk); + return found; +} + +/** + * @brief Sub-test: basic produce → share-consume → verify metrics flow. + * + * Produces a batch of messages to a fresh topic, drives a share consumer + * over them so the telemetry sampling sites fire, then consumes from the + * telemetry topic and asserts every expected share-consumer metric name + * appears at least once. + */ +static void do_test_produce_share_consume_verify_metrics(void) { + const char *data_topic = test_mk_topic_name("0190-data", 1); + const char *group_id = test_mk_topic_name("0190-share-grp", 0); + seen_metrics_t seen = {0}; + + SUB_TEST(); + + produce_and_share_consume(data_topic, group_id); + + consume_telemetry_topic(telemetry_topic(), 30, &seen); + verify_metrics(&seen); + + SUB_TEST_PASS(); +} + + +int main_0190_share_consumer_telemetry(int argc, char **argv) { + test_timeout_set(180); + + if (!telemetry_infra_available(telemetry_topic())) { + TEST_SKIP( + "telemetry topic '%s' not found — " + "OTel/plugin pipeline not configured for this run\n", + telemetry_topic()); + return 0; + } + + do_test_produce_share_consume_verify_metrics(); + + return 0; +} diff --git a/lib/librdkafka-2.10.1/tests/1000-unktopic.c b/lib/librdkafka-2.15.0/tests/1000-unktopic.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/1000-unktopic.c rename to lib/librdkafka-2.15.0/tests/1000-unktopic.c diff --git a/lib/librdkafka-2.10.1/tests/8000-idle.cpp b/lib/librdkafka-2.15.0/tests/8000-idle.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/8000-idle.cpp rename to lib/librdkafka-2.15.0/tests/8000-idle.cpp diff --git a/lib/librdkafka-2.10.1/tests/8001-fetch_from_follower_mock_manual.c b/lib/librdkafka-2.15.0/tests/8001-fetch_from_follower_mock_manual.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/8001-fetch_from_follower_mock_manual.c rename to lib/librdkafka-2.15.0/tests/8001-fetch_from_follower_mock_manual.c diff --git a/lib/librdkafka-2.10.1/tests/CMakeLists.txt b/lib/librdkafka-2.15.0/tests/CMakeLists.txt similarity index 80% rename from lib/librdkafka-2.10.1/tests/CMakeLists.txt rename to lib/librdkafka-2.15.0/tests/CMakeLists.txt index da9c32c7738..0f6a0650fe6 100644 --- a/lib/librdkafka-2.10.1/tests/CMakeLists.txt +++ b/lib/librdkafka-2.15.0/tests/CMakeLists.txt @@ -137,10 +137,35 @@ set( 0144-idempotence_mock.c 0145-pause_resume_mock.c 0146-metadata_mock.c + 0147-consumer_group_consumer_mock.c + 0148-offset_fetch_commit_error_mock.c 0149-broker-same-host-port.c 0150-telemetry_mock.c 0151-purge-brokers.c 0152-rebootstrap.c + 0153-memberid.c + 0155-share_group_heartbeat_mock.c + 0156-share_consumer_fetch_mock.c + 0157-share_consumer_ack_mock.c + 0158-share_consumer_transactions_mock.c + 0170-share_consumer_subscription.c + 0171-share_consumer_consume.c + 0172-share_consumer_acknowledge.c + 0173-share_consumer_commit_async.c + 0174-share_consumer_concurrency.c + 0175-share_consumer_groups.c + 0176-share_consumer_commit_sync.c + 0177-share_consumer_transactions.c + 0178-share_consumer_close.c + 0179-share_consumer_destroy.c + 0180-share_consumer_config.c + 0181-share_consumer_topic_delete.c + 0182-share_consumer_error_handling_mock.c + 0183-share_consumer_leader_change_mock.c + 0184-share_consumer_topic_recreate.c + 0185-share_consumer_max_poll_interval.c + 0186-share_consumer_fatal_error.c + 0190-share_consumer_telemetry.c 8000-idle.cpp 8001-fetch_from_follower_mock_manual.c test.c @@ -160,6 +185,7 @@ target_link_libraries(test-runner PUBLIC rdkafka++) add_test(NAME RdKafkaTestInParallel COMMAND test-runner -p5) add_test(NAME RdKafkaTestSequentially COMMAND test-runner -p1) add_test(NAME RdKafkaTestBrokerLess COMMAND test-runner -p5 -l) +add_test(NAME RdKafkaTestBrokerLessQuick COMMAND test-runner -p5 -l -Q) if(NOT WIN32 AND NOT APPLE) set(tests_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/lib/librdkafka-2.10.1/tests/LibrdkafkaTestApp.py b/lib/librdkafka-2.15.0/tests/LibrdkafkaTestApp.py similarity index 91% rename from lib/librdkafka-2.10.1/tests/LibrdkafkaTestApp.py rename to lib/librdkafka-2.15.0/tests/LibrdkafkaTestApp.py index 00dc3027e32..4a9816b71ed 100644 --- a/lib/librdkafka-2.10.1/tests/LibrdkafkaTestApp.py +++ b/lib/librdkafka-2.15.0/tests/LibrdkafkaTestApp.py @@ -88,12 +88,28 @@ def __init__(self, cluster, version, conf=None, self.env_add( 'EXPIRED_TOKEN_OIDC_URL', oidc.conf.get('expired_url')) + self.env_add( + 'EXPIRED_TOKEN_OIDC_URL', + oidc.conf.get('expired_url')) + self.env['OAUTHBEARER_CLIENT_PRIVATE_KEY'] = \ + oidc.conf['sasl_oauthbearer_client_private_key_path'] + self.env['OAUTHBEARER_CLIENT_PRIVATE_KEY_ENCRYPTED'] = \ + oidc.conf[ + 'sasl_oauthbearer_client_private_key_encrypted_path'] # noqa: E501 + self.env['OAUTHBEARER_CLIENT_PRIVATE_KEY_PASSWORD'] = \ + oidc.conf[ + 'sasl_oauthbearer_client_private_key_password'] else: conf_blob.append( 'enable.sasl.oauthbearer.unsecure.jwt=true\n') - conf_blob.append( - 'sasl.oauthbearer.config=%s\n' % - self.conf.get('sasl_oauthbearer_config')) + if not self.conf.get('sasl_oauthbearer_config'): + conf_blob.append( + 'sasl.oauthbearer.config=scope=requiredScope' + ' principal=admin\n') + else: + conf_blob.append( + 'sasl.oauthbearer.config=%s\n' % + self.conf.get('sasl_oauthbearer_config')) elif mech == 'GSSAPI': self.security_protocol = 'SASL_PLAINTEXT' diff --git a/lib/librdkafka-2.10.1/tests/Makefile b/lib/librdkafka-2.15.0/tests/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/tests/Makefile rename to lib/librdkafka-2.15.0/tests/Makefile diff --git a/lib/librdkafka-2.10.1/tests/README.md b/lib/librdkafka-2.15.0/tests/README.md similarity index 97% rename from lib/librdkafka-2.10.1/tests/README.md rename to lib/librdkafka-2.15.0/tests/README.md index 4d2c011ad31..9ab6f8832db 100644 --- a/lib/librdkafka-2.10.1/tests/README.md +++ b/lib/librdkafka-2.15.0/tests/README.md @@ -180,6 +180,11 @@ be it `make`, `run-test.sh`, `until-fail.sh`, etc. * `TEST_DEBUG=...` - this will automatically set the `debug` config property of all instantiated clients to the value. E.g.. `TEST_DEBUG=broker,protocol TESTS=0001 make` + * `TESTS_TO_DEBUG=0nnn,0mmm` - when used with `TEST_DEBUG`, limits debug + logging to only the specified comma-separated test numbers. + This is useful when running multiple tests but only wanting + debug output from specific ones. + E.g., `TESTS_TO_DEBUG=0172,0171 TEST_DEBUG=all make` * `TEST_LEVEL=n` - controls the `TEST_SAY()` output level, a higher number yields more test output. Default level is 2. * `RD_UT_TEST=name` - only run unittest containing `name`, should be used diff --git a/lib/librdkafka-2.10.1/tests/autotest.sh b/lib/librdkafka-2.15.0/tests/autotest.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/autotest.sh rename to lib/librdkafka-2.15.0/tests/autotest.sh diff --git a/lib/librdkafka-2.10.1/tests/backtrace.gdb b/lib/librdkafka-2.15.0/tests/backtrace.gdb similarity index 100% rename from lib/librdkafka-2.10.1/tests/backtrace.gdb rename to lib/librdkafka-2.15.0/tests/backtrace.gdb diff --git a/lib/librdkafka-2.10.1/tests/broker_version_tests.py b/lib/librdkafka-2.15.0/tests/broker_version_tests.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/broker_version_tests.py rename to lib/librdkafka-2.15.0/tests/broker_version_tests.py diff --git a/lib/librdkafka-2.10.1/tests/buildbox.sh b/lib/librdkafka-2.15.0/tests/buildbox.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/buildbox.sh rename to lib/librdkafka-2.15.0/tests/buildbox.sh diff --git a/lib/librdkafka-2.10.1/tests/cleanup-checker-tests.sh b/lib/librdkafka-2.15.0/tests/cleanup-checker-tests.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/cleanup-checker-tests.sh rename to lib/librdkafka-2.15.0/tests/cleanup-checker-tests.sh diff --git a/lib/librdkafka-2.10.1/tests/cluster_testing.py b/lib/librdkafka-2.15.0/tests/cluster_testing.py similarity index 84% rename from lib/librdkafka-2.10.1/tests/cluster_testing.py rename to lib/librdkafka-2.15.0/tests/cluster_testing.py index d3189f1cdb9..1d91acaea47 100755 --- a/lib/librdkafka-2.10.1/tests/cluster_testing.py +++ b/lib/librdkafka-2.15.0/tests/cluster_testing.py @@ -92,13 +92,25 @@ def __init__(self, version, conf={}, num_brokers=3, debug=False, # fetch-from-follower if version_as_list(version) >= [2, 4, 0]: curr_conf = defconf_curr.get('conf', list()) + extra_conf = [ + 'broker.rack=RACK${appid}', + 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector' # noqa: E501 + ] + # Share consumer support (KIP-932) requires + # share coordinator state topic settings for + # single-broker test clusters. + if version_as_list(version) >= [4, 2, 0]: + extra_conf += [ + 'share.coordinator.state.topic.replication.factor=%d' % min(num_brokers, 3), # noqa: E501 + 'share.coordinator.state.topic.min.isr=%d' % min(num_brokers, 2), # noqa: E501 + 'group.share.min.record.lock.duration.ms=1000', + 'transaction.state.log.replication.factor=%d' % min(num_brokers, 3), # noqa: E501 + 'transaction.state.log.min.isr=%d' % min(num_brokers, 2), # noqa: E501 + ] defconf_curr.update( { - 'conf': [ - 'broker.rack=RACK${appid}', - 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector' # noqa: E501 - ] + curr_conf - }) # noqa: E501 + 'conf': extra_conf + curr_conf + }) print('conf broker', str(n), ': ', defconf_curr) self.brokers.append(KafkaBrokerApp(self, defconf_curr)) diff --git a/lib/librdkafka-2.10.1/tests/delete-test-topics.sh b/lib/librdkafka-2.15.0/tests/delete-test-topics.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/delete-test-topics.sh rename to lib/librdkafka-2.15.0/tests/delete-test-topics.sh diff --git a/lib/librdkafka-2.15.0/tests/fixtures/oauthbearer/jwt_assertion_template.json b/lib/librdkafka-2.15.0/tests/fixtures/oauthbearer/jwt_assertion_template.json new file mode 100644 index 00000000000..15f27ad4a87 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/fixtures/oauthbearer/jwt_assertion_template.json @@ -0,0 +1,10 @@ +{ + "header": {}, + "payload": { + "sub": "testuser-template", + "aud": "https://token_endpoint/token", + "iss": "my_service", + "target_audience": "kafka-cluster", + "jti":"94232d62-8b0c-406b-ba12-f92d7f522253" + } +} diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/.gitignore b/lib/librdkafka-2.15.0/tests/fixtures/ssl/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/.gitignore rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/.gitignore diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/Makefile b/lib/librdkafka-2.15.0/tests/fixtures/ssl/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/Makefile rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/Makefile diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/README.md b/lib/librdkafka-2.15.0/tests/fixtures/ssl/README.md similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/README.md rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/README.md diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/client.keystore.intermediate.p12 b/lib/librdkafka-2.15.0/tests/fixtures/ssl/client.keystore.intermediate.p12 similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/client.keystore.intermediate.p12 rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/client.keystore.intermediate.p12 diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/client.keystore.p12 b/lib/librdkafka-2.15.0/tests/fixtures/ssl/client.keystore.p12 similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/client.keystore.p12 rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/client.keystore.p12 diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.certificate.intermediate.pem b/lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.certificate.intermediate.pem similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.certificate.intermediate.pem rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.certificate.intermediate.pem diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.certificate.pem b/lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.certificate.pem similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.certificate.pem rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.certificate.pem diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.intermediate.key b/lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.intermediate.key similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.intermediate.key rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.intermediate.key diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.key b/lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.key similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/client2.key rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/client2.key diff --git a/lib/librdkafka-2.10.1/tests/fixtures/ssl/create_keys.sh b/lib/librdkafka-2.15.0/tests/fixtures/ssl/create_keys.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/fixtures/ssl/create_keys.sh rename to lib/librdkafka-2.15.0/tests/fixtures/ssl/create_keys.sh diff --git a/lib/librdkafka-2.10.1/tests/fuzzers/.gitignore b/lib/librdkafka-2.15.0/tests/fuzzers/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/tests/fuzzers/.gitignore rename to lib/librdkafka-2.15.0/tests/fuzzers/.gitignore diff --git a/lib/librdkafka-2.10.1/tests/fuzzers/Makefile b/lib/librdkafka-2.15.0/tests/fuzzers/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/tests/fuzzers/Makefile rename to lib/librdkafka-2.15.0/tests/fuzzers/Makefile diff --git a/lib/librdkafka-2.10.1/tests/fuzzers/README.md b/lib/librdkafka-2.15.0/tests/fuzzers/README.md similarity index 100% rename from lib/librdkafka-2.10.1/tests/fuzzers/README.md rename to lib/librdkafka-2.15.0/tests/fuzzers/README.md diff --git a/lib/librdkafka-2.10.1/tests/fuzzers/fuzz_regex.c b/lib/librdkafka-2.15.0/tests/fuzzers/fuzz_regex.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/fuzzers/fuzz_regex.c rename to lib/librdkafka-2.15.0/tests/fuzzers/fuzz_regex.c diff --git a/lib/librdkafka-2.10.1/tests/fuzzers/helpers.h b/lib/librdkafka-2.15.0/tests/fuzzers/helpers.h similarity index 100% rename from lib/librdkafka-2.10.1/tests/fuzzers/helpers.h rename to lib/librdkafka-2.15.0/tests/fuzzers/helpers.h diff --git a/lib/librdkafka-2.10.1/tests/gen-ssl-certs.sh b/lib/librdkafka-2.15.0/tests/gen-ssl-certs.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/gen-ssl-certs.sh rename to lib/librdkafka-2.15.0/tests/gen-ssl-certs.sh diff --git a/lib/librdkafka-2.10.1/tests/interactive_broker_version.py b/lib/librdkafka-2.15.0/tests/interactive_broker_version.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/interactive_broker_version.py rename to lib/librdkafka-2.15.0/tests/interactive_broker_version.py diff --git a/lib/librdkafka-2.10.1/tests/interceptor_test/.gitignore b/lib/librdkafka-2.15.0/tests/interceptor_test/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/tests/interceptor_test/.gitignore rename to lib/librdkafka-2.15.0/tests/interceptor_test/.gitignore diff --git a/lib/librdkafka-2.10.1/tests/interceptor_test/CMakeLists.txt b/lib/librdkafka-2.15.0/tests/interceptor_test/CMakeLists.txt similarity index 100% rename from lib/librdkafka-2.10.1/tests/interceptor_test/CMakeLists.txt rename to lib/librdkafka-2.15.0/tests/interceptor_test/CMakeLists.txt diff --git a/lib/librdkafka-2.10.1/tests/interceptor_test/Makefile b/lib/librdkafka-2.15.0/tests/interceptor_test/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/tests/interceptor_test/Makefile rename to lib/librdkafka-2.15.0/tests/interceptor_test/Makefile diff --git a/lib/librdkafka-2.10.1/tests/interceptor_test/interceptor_test.c b/lib/librdkafka-2.15.0/tests/interceptor_test/interceptor_test.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/interceptor_test/interceptor_test.c rename to lib/librdkafka-2.15.0/tests/interceptor_test/interceptor_test.c diff --git a/lib/librdkafka-2.10.1/tests/interceptor_test/interceptor_test.h b/lib/librdkafka-2.15.0/tests/interceptor_test/interceptor_test.h similarity index 100% rename from lib/librdkafka-2.10.1/tests/interceptor_test/interceptor_test.h rename to lib/librdkafka-2.15.0/tests/interceptor_test/interceptor_test.h diff --git a/lib/librdkafka-2.10.1/tests/java/.gitignore b/lib/librdkafka-2.15.0/tests/java/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/.gitignore rename to lib/librdkafka-2.15.0/tests/java/.gitignore diff --git a/lib/librdkafka-2.10.1/tests/java/IncrementalRebalanceCli.java b/lib/librdkafka-2.15.0/tests/java/IncrementalRebalanceCli.java similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/IncrementalRebalanceCli.java rename to lib/librdkafka-2.15.0/tests/java/IncrementalRebalanceCli.java diff --git a/lib/librdkafka-2.10.1/tests/java/Makefile b/lib/librdkafka-2.15.0/tests/java/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/Makefile rename to lib/librdkafka-2.15.0/tests/java/Makefile diff --git a/lib/librdkafka-2.10.1/tests/java/Murmur2Cli.java b/lib/librdkafka-2.15.0/tests/java/Murmur2Cli.java similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/Murmur2Cli.java rename to lib/librdkafka-2.15.0/tests/java/Murmur2Cli.java diff --git a/lib/librdkafka-2.10.1/tests/java/README.md b/lib/librdkafka-2.15.0/tests/java/README.md similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/README.md rename to lib/librdkafka-2.15.0/tests/java/README.md diff --git a/lib/librdkafka-2.10.1/tests/java/TransactionProducerCli.java b/lib/librdkafka-2.15.0/tests/java/TransactionProducerCli.java similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/TransactionProducerCli.java rename to lib/librdkafka-2.15.0/tests/java/TransactionProducerCli.java diff --git a/lib/librdkafka-2.10.1/tests/java/run-class.sh b/lib/librdkafka-2.15.0/tests/java/run-class.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/java/run-class.sh rename to lib/librdkafka-2.15.0/tests/java/run-class.sh diff --git a/lib/librdkafka-2.10.1/tests/librdkafka.suppressions b/lib/librdkafka-2.15.0/tests/librdkafka.suppressions similarity index 100% rename from lib/librdkafka-2.10.1/tests/librdkafka.suppressions rename to lib/librdkafka-2.15.0/tests/librdkafka.suppressions diff --git a/lib/librdkafka-2.10.1/tests/lz4_manual_test.sh b/lib/librdkafka-2.15.0/tests/lz4_manual_test.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/lz4_manual_test.sh rename to lib/librdkafka-2.15.0/tests/lz4_manual_test.sh diff --git a/lib/librdkafka-2.10.1/tests/multi-broker-version-test.sh b/lib/librdkafka-2.15.0/tests/multi-broker-version-test.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/multi-broker-version-test.sh rename to lib/librdkafka-2.15.0/tests/multi-broker-version-test.sh diff --git a/lib/librdkafka-2.10.1/tests/parse-refcnt.sh b/lib/librdkafka-2.15.0/tests/parse-refcnt.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/parse-refcnt.sh rename to lib/librdkafka-2.15.0/tests/parse-refcnt.sh diff --git a/lib/librdkafka-2.10.1/tests/performance_plot.py b/lib/librdkafka-2.15.0/tests/performance_plot.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/performance_plot.py rename to lib/librdkafka-2.15.0/tests/performance_plot.py diff --git a/lib/librdkafka-2.10.1/tests/plugin_test/Makefile b/lib/librdkafka-2.15.0/tests/plugin_test/Makefile similarity index 100% rename from lib/librdkafka-2.10.1/tests/plugin_test/Makefile rename to lib/librdkafka-2.15.0/tests/plugin_test/Makefile diff --git a/lib/librdkafka-2.10.1/tests/plugin_test/plugin_test.c b/lib/librdkafka-2.15.0/tests/plugin_test/plugin_test.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/plugin_test/plugin_test.c rename to lib/librdkafka-2.15.0/tests/plugin_test/plugin_test.c diff --git a/lib/librdkafka-2.15.0/tests/requirements.txt b/lib/librdkafka-2.15.0/tests/requirements.txt new file mode 100644 index 00000000000..21b2cfeb4c2 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/requirements.txt @@ -0,0 +1,2 @@ +trivup/trivup-0.15.0.tar.gz +jsoncomment diff --git a/lib/librdkafka-2.10.1/tests/run-all-tests.sh b/lib/librdkafka-2.15.0/tests/run-all-tests.sh similarity index 90% rename from lib/librdkafka-2.10.1/tests/run-all-tests.sh rename to lib/librdkafka-2.15.0/tests/run-all-tests.sh index 8708913033d..0e5ecbcfe8a 100755 --- a/lib/librdkafka-2.10.1/tests/run-all-tests.sh +++ b/lib/librdkafka-2.15.0/tests/run-all-tests.sh @@ -14,8 +14,8 @@ set -e # - no assertions # - following AK and CP versions -export TEST_KAFKA_GIT_REF=${TEST_KAFKA_GIT_REF:-4.0.0} -export TEST_CP_VERSION=${TEST_CP_VERSION:-7.9.0} +export TEST_KAFKA_GIT_REF=${TEST_KAFKA_GIT_REF:-4.3.0} +export TEST_CP_VERSION=${TEST_CP_VERSION:-8.0.0} TEST_SSL_ARG="" TEST_SSL_INTERMEDIATE_CA_ARG="" @@ -54,7 +54,9 @@ if [ ! -z $TEST_PARALLEL ]; then TEST_PARALLEL_ARG="-p$TEST_PARALLEL" fi if [ ! -z $TEST_CONF ]; then - TEST_CONF_ARG="--conf '$TEST_CONF'" + TEST_CONF_ARG="--conf $TEST_CONF" +else + TEST_CONF_ARG="--conf [\"group.share.min.record.lock.duration.ms=1000\"]" fi if [ ! -z $TEST_ENV_VARIABLES ]; then IFS=',' read -ra TEST_ENV_VARIABLES_ARRAY <<< "$TEST_ENV_VARIABLES" diff --git a/lib/librdkafka-2.10.1/tests/run-consumer-tests.sh b/lib/librdkafka-2.15.0/tests/run-consumer-tests.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/run-consumer-tests.sh rename to lib/librdkafka-2.15.0/tests/run-consumer-tests.sh diff --git a/lib/librdkafka-2.10.1/tests/run-producer-tests.sh b/lib/librdkafka-2.15.0/tests/run-producer-tests.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/run-producer-tests.sh rename to lib/librdkafka-2.15.0/tests/run-producer-tests.sh diff --git a/lib/librdkafka-2.10.1/tests/run-test-batches.py b/lib/librdkafka-2.15.0/tests/run-test-batches.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/run-test-batches.py rename to lib/librdkafka-2.15.0/tests/run-test-batches.py diff --git a/lib/librdkafka-2.10.1/tests/run-test.sh b/lib/librdkafka-2.15.0/tests/run-test.sh similarity index 92% rename from lib/librdkafka-2.10.1/tests/run-test.sh rename to lib/librdkafka-2.15.0/tests/run-test.sh index 2f531c61f05..c3b52dd599e 100755 --- a/lib/librdkafka-2.10.1/tests/run-test.sh +++ b/lib/librdkafka-2.15.0/tests/run-test.sh @@ -48,8 +48,9 @@ VALGRIND_ARGS="--error-exitcode=3" # Enable vgdb on valgrind errors. #VALGRIND_ARGS="$VALGRIND_ARGS --vgdb-error=1" -# Exit valgrind on first error -VALGRIND_ARGS="$VALGRIND_ARGS --exit-on-first-error=yes" +# Report all errors instead of stopping at the first one, so a leading +# "still reachable" block (e.g. from OpenSSL) doesn't hide later real leaks. +VALGRIND_ARGS="$VALGRIND_ARGS --exit-on-first-error=no" export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../src:../src-cpp export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:../src:../src-cpp @@ -62,7 +63,7 @@ for mode in $MODES; do case "$mode" in valgrind) valgrind $VALGRIND_ARGS --leak-check=full --show-leak-kinds=all \ - --errors-for-leak-kinds=all \ + --errors-for-leak-kinds=definite,possible \ --track-origins=yes \ --track-fds=yes \ $SUPP $GEN_SUPP \ diff --git a/lib/librdkafka-2.10.1/tests/rusage.c b/lib/librdkafka-2.15.0/tests/rusage.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/rusage.c rename to lib/librdkafka-2.15.0/tests/rusage.c diff --git a/lib/librdkafka-2.10.1/tests/sasl_test.py b/lib/librdkafka-2.15.0/tests/sasl_test.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/sasl_test.py rename to lib/librdkafka-2.15.0/tests/sasl_test.py diff --git a/lib/librdkafka-2.10.1/tests/scenarios/README.md b/lib/librdkafka-2.15.0/tests/scenarios/README.md similarity index 100% rename from lib/librdkafka-2.10.1/tests/scenarios/README.md rename to lib/librdkafka-2.15.0/tests/scenarios/README.md diff --git a/lib/librdkafka-2.10.1/tests/scenarios/ak23.json b/lib/librdkafka-2.15.0/tests/scenarios/ak23.json similarity index 100% rename from lib/librdkafka-2.10.1/tests/scenarios/ak23.json rename to lib/librdkafka-2.15.0/tests/scenarios/ak23.json diff --git a/lib/librdkafka-2.10.1/tests/scenarios/default.json b/lib/librdkafka-2.15.0/tests/scenarios/default.json similarity index 100% rename from lib/librdkafka-2.10.1/tests/scenarios/default.json rename to lib/librdkafka-2.15.0/tests/scenarios/default.json diff --git a/lib/librdkafka-2.10.1/tests/scenarios/noautocreate.json b/lib/librdkafka-2.15.0/tests/scenarios/noautocreate.json similarity index 100% rename from lib/librdkafka-2.10.1/tests/scenarios/noautocreate.json rename to lib/librdkafka-2.15.0/tests/scenarios/noautocreate.json diff --git a/lib/librdkafka-2.15.0/tests/share_telemetry_verify/share-otel-collector-config.yaml b/lib/librdkafka-2.15.0/tests/share_telemetry_verify/share-otel-collector-config.yaml new file mode 100644 index 00000000000..9858c466af4 --- /dev/null +++ b/lib/librdkafka-2.15.0/tests/share_telemetry_verify/share-otel-collector-config.yaml @@ -0,0 +1,28 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: + send_batch_max_size: 100 + send_batch_size: 10 + timeout: 5s + +exporters: + kafka: + brokers: + - host.docker.internal:19092 + metrics: + topic: ${env:TELEMETRY_TOPIC} + encoding: otlp_proto + debug: + verbosity: basic + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [kafka, debug] diff --git a/lib/librdkafka-2.10.1/tests/sockem.c b/lib/librdkafka-2.15.0/tests/sockem.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/sockem.c rename to lib/librdkafka-2.15.0/tests/sockem.c diff --git a/lib/librdkafka-2.10.1/tests/sockem.h b/lib/librdkafka-2.15.0/tests/sockem.h similarity index 100% rename from lib/librdkafka-2.10.1/tests/sockem.h rename to lib/librdkafka-2.15.0/tests/sockem.h diff --git a/lib/librdkafka-2.10.1/tests/sockem_ctrl.c b/lib/librdkafka-2.15.0/tests/sockem_ctrl.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/sockem_ctrl.c rename to lib/librdkafka-2.15.0/tests/sockem_ctrl.c diff --git a/lib/librdkafka-2.10.1/tests/sockem_ctrl.h b/lib/librdkafka-2.15.0/tests/sockem_ctrl.h similarity index 100% rename from lib/librdkafka-2.10.1/tests/sockem_ctrl.h rename to lib/librdkafka-2.15.0/tests/sockem_ctrl.h diff --git a/lib/librdkafka-2.10.1/tests/test.c b/lib/librdkafka-2.15.0/tests/test.c similarity index 90% rename from lib/librdkafka-2.10.1/tests/test.c rename to lib/librdkafka-2.15.0/tests/test.c index cbbe1f91a3c..664376afda0 100644 --- a/lib/librdkafka-2.10.1/tests/test.c +++ b/lib/librdkafka-2.15.0/tests/test.c @@ -45,29 +45,43 @@ * is built from within the librdkafka source tree and thus differs. */ #include "rdkafka.h" +/** + * Local definition of rd_kafka_share_s to access rkshare_rk + * without pulling in rdkafka_int.h (which causes Windows link errors + * due to internal inline functions referencing unexported symbols). + * + * TODO: Replace with a proper public API (e.g. + * test_share_consumer_get_rk()) so tests don't depend on the + * internal struct layout. + */ +struct rd_kafka_share_s { + rd_kafka_t *rkshare_rk; +}; + + int test_level = 2; int test_seed = 0; -char test_mode[64] = "bare"; -char test_scenario[64] = "default"; -static volatile sig_atomic_t test_exit = 0; -static char test_topic_prefix[128] = "rdkafkatest"; -static int test_topic_random = 0; -int tests_running_cnt = 0; -int test_concurrent_max = 5; -int test_assert_on_fail = 0; -double test_timeout_multiplier = 1.0; -static char *test_sql_cmd = NULL; -int test_session_timeout_ms = 6000; -static const char *test_consumer_group_protocol_str = NULL; +char test_mode[64] = "bare"; +char test_scenario[64] = "default"; +static volatile sig_atomic_t test_exit = 0; +static char test_topic_prefix[128] = "rdkafkatest"; +static int test_topic_random = 0; +int tests_running_cnt = 0; +int test_concurrent_max = 5; +int test_assert_on_fail = 0; +double test_timeout_multiplier = 1.0; +static char *test_sql_cmd = NULL; +int test_session_timeout_ms = 6000; +static char *test_consumer_group_protocol_str = NULL; int test_broker_version; -static const char *test_broker_version_str = "2.4.0.0"; -int test_flags = 0; -int test_neg_flags = TEST_F_KNOWN_ISSUE; +static char *test_broker_version_str = "2.4.0.0"; +int test_flags = 0; +int test_neg_flags = TEST_F_KNOWN_ISSUE; /* run delete-test-topics.sh between each test (when concurrent_max = 1) */ static int test_delete_topics_between = 0; -static const char *test_git_version = "HEAD"; -static const char *test_sockem_conf = ""; +static char *test_git_version = "HEAD"; +static char *test_sockem_conf = ""; int test_on_ci = 0; /* Tests are being run on CI, be more forgiving * with regards to timeouts, etc. */ int test_quick = 0; /** Run tests quickly */ @@ -77,10 +91,10 @@ int test_rusage = 0; /**< Check resource usage */ * >1.0: CPU is slower than base line system, * <1.0: CPU is faster than base line system. */ double test_rusage_cpu_calibration = 1.0; -static const char *tests_to_run = NULL; /* all */ -static const char *skip_tests_till = NULL; /* all */ -static const char *subtests_to_run = NULL; /* all */ -static const char *tests_to_skip = NULL; /* none */ +static char *tests_to_run = NULL; /* all */ +static char *skip_tests_till = NULL; /* all */ +static char *subtests_to_run = NULL; /* all */ +static char *tests_to_skip = NULL; /* none */ int test_write_report = 0; /**< Write test report file */ static int show_summary = 1; @@ -158,6 +172,7 @@ _TEST_DECL(0045_subscribe_update_non_exist_and_partchange); _TEST_DECL(0045_subscribe_update_mock); _TEST_DECL(0045_subscribe_update_racks_mock); _TEST_DECL(0045_resubscribe_with_regex); +_TEST_DECL(0045_subscribe_many_updates); _TEST_DECL(0046_rkt_cache); _TEST_DECL(0047_partial_buf_tmout); _TEST_DECL(0048_partitioner); @@ -264,10 +279,41 @@ _TEST_DECL(0143_exponential_backoff_mock); _TEST_DECL(0144_idempotence_mock); _TEST_DECL(0145_pause_resume_mock); _TEST_DECL(0146_metadata_mock); +_TEST_DECL(0147_consumer_group_consumer_mock); +_TEST_DECL(0148_offset_fetch_commit_error_mock); _TEST_DECL(0149_broker_same_host_port_mock); _TEST_DECL(0150_telemetry_mock); _TEST_DECL(0151_purge_brokers_mock); _TEST_DECL(0152_rebootstrap_local); +_TEST_DECL(0153_memberid); +_TEST_DECL(0155_share_group_heartbeat_mock); +_TEST_DECL(0156_share_consumer_fetch_mock); +_TEST_DECL(0157_share_consumer_ack_mock); +_TEST_DECL(0158_share_consumer_transactions_mock); +_TEST_DECL(0170_share_consumer_subscription); +_TEST_DECL(0171_share_consumer_consume); +_TEST_DECL(0172_share_consumer_acknowledge); +_TEST_DECL(0173_share_consumer_commit_async_local); +_TEST_DECL(0173_share_consumer_commit_async); +_TEST_DECL(0174_share_consumer_concurrency); +_TEST_DECL(0175_share_consumer_groups); +_TEST_DECL(0176_share_consumer_commit_sync); +_TEST_DECL(0176_share_consumer_commit_sync_local); +_TEST_DECL(0177_share_consumer_transactions); +_TEST_DECL(0178_share_consumer_close); +_TEST_DECL(0178_share_consumer_close_local); +_TEST_DECL(0179_share_consumer_destroy); +_TEST_DECL(0179_share_consumer_destroy_local); +_TEST_DECL(0180_share_consumer_config); +_TEST_DECL(0180_share_consumer_config_local); +_TEST_DECL(0181_share_consumer_topic_delete); +_TEST_DECL(0182_share_consumer_error_handling_mock); +_TEST_DECL(0183_share_consumer_leader_change_mock); +_TEST_DECL(0184_share_consumer_topic_recreate); +_TEST_DECL(0184_share_consumer_topic_recreate_local); +_TEST_DECL(0185_share_consumer_max_poll_interval); +_TEST_DECL(0186_share_consumer_fatal_error); +_TEST_DECL(0190_share_consumer_telemetry); /* Manual tests */ _TEST_DECL(8000_idle); @@ -385,6 +431,7 @@ struct test tests[] = { _TEST(0045_subscribe_update_mock, TEST_F_LOCAL), _TEST(0045_subscribe_update_racks_mock, TEST_F_LOCAL), _TEST(0045_resubscribe_with_regex, 0, TEST_BRKVER(0, 9, 0, 0)), + _TEST(0045_subscribe_many_updates, 0, TEST_BRKVER(0, 10, 2, 0)), _TEST(0046_rkt_cache, TEST_F_LOCAL), _TEST(0047_partial_buf_tmout, TEST_F_KNOWN_ISSUE), _TEST(0048_partitioner, @@ -528,11 +575,43 @@ struct test tests[] = { _TEST(0144_idempotence_mock, TEST_F_LOCAL, TEST_BRKVER(0, 11, 0, 0)), _TEST(0145_pause_resume_mock, TEST_F_LOCAL), _TEST(0146_metadata_mock, TEST_F_LOCAL), + _TEST(0147_consumer_group_consumer_mock, TEST_F_LOCAL), + _TEST(0148_offset_fetch_commit_error_mock, TEST_F_LOCAL), _TEST(0149_broker_same_host_port_mock, TEST_F_LOCAL), _TEST(0150_telemetry_mock, 0), _TEST(0151_purge_brokers_mock, TEST_F_LOCAL), _TEST(0152_rebootstrap_local, TEST_F_LOCAL), - + _TEST(0153_memberid, TEST_F_LOCAL), + _TEST(0155_share_group_heartbeat_mock, TEST_F_LOCAL), + _TEST(0156_share_consumer_fetch_mock, TEST_F_LOCAL), + _TEST(0157_share_consumer_ack_mock, TEST_F_LOCAL), + _TEST(0158_share_consumer_transactions_mock, TEST_F_LOCAL), + _TEST(0170_share_consumer_subscription, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0171_share_consumer_consume, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0172_share_consumer_acknowledge, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0173_share_consumer_commit_async_local, TEST_F_LOCAL), + _TEST(0173_share_consumer_commit_async, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0174_share_consumer_concurrency, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0175_share_consumer_groups, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0176_share_consumer_commit_sync, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0176_share_consumer_commit_sync_local, TEST_F_LOCAL), + _TEST(0177_share_consumer_transactions, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0178_share_consumer_close, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0178_share_consumer_close_local, TEST_F_LOCAL), + _TEST(0179_share_consumer_destroy, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0179_share_consumer_destroy_local, TEST_F_LOCAL), + _TEST(0180_share_consumer_config, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0180_share_consumer_config_local, TEST_F_LOCAL), + _TEST(0181_share_consumer_topic_delete, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0182_share_consumer_error_handling_mock, TEST_F_LOCAL), + _TEST(0183_share_consumer_leader_change_mock, TEST_F_LOCAL), + _TEST(0184_share_consumer_topic_recreate, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0184_share_consumer_topic_recreate_local, TEST_F_LOCAL), + _TEST(0185_share_consumer_max_poll_interval, 0, TEST_BRKVER(4, 2, 0, 0)), + _TEST(0186_share_consumer_fatal_error, TEST_F_LOCAL), + _TEST(0190_share_consumer_telemetry, + TEST_F_MANUAL, + TEST_BRKVER(4, 2, 0, 0)), /* Manual tests */ _TEST(8000_idle, TEST_F_MANUAL), @@ -556,6 +635,14 @@ static void test_socket_add(struct test *test, sockem_t *skm) { TEST_UNLOCK(); } +void *test_socket_find(struct test *test, sockem_t *skm) { + void *ret; + TEST_LOCK(); + ret = rd_list_find(&test->sockets, skm, rd_list_cmp_ptr); + TEST_UNLOCK(); + return ret; +} + static void test_socket_del(struct test *test, sockem_t *skm, int do_lock) { if (do_lock) TEST_LOCK(); @@ -769,7 +856,8 @@ static void test_init(void) { if ((tmp = test_getenv("TEST_SCENARIO", NULL))) strncpy(test_scenario, tmp, sizeof(test_scenario) - 1); if ((tmp = test_getenv("TEST_SOCKEM", NULL))) - test_sockem_conf = tmp; + test_sockem_conf = (char *)tmp; + test_sockem_conf = rd_strdup(test_sockem_conf); if ((tmp = test_getenv("TEST_SEED", NULL))) seed = atoi(tmp); else @@ -785,7 +873,10 @@ static void test_init(void) { } } test_consumer_group_protocol_str = - test_getenv("TEST_CONSUMER_GROUP_PROTOCOL", NULL); + (char *)test_getenv("TEST_CONSUMER_GROUP_PROTOCOL", NULL); + if (test_consumer_group_protocol_str) + test_consumer_group_protocol_str = + rd_strdup(test_consumer_group_protocol_str); #ifdef _WIN32 @@ -994,10 +1085,81 @@ const char *test_getenv(const char *env, const char *def) { return rd_getenv(env, def); } +/** + * @brief Check if debug logging should be enabled for the current test. + * + * If TESTS_TO_DEBUG environment variable is set with comma-separated test + * numbers (e.g., "0172,0171"), only enable debug for those tests. + * Otherwise, enable debug for all tests if TEST_DEBUG is set. + * + * @returns 1 if debug should be enabled, 0 otherwise. + */ +static int test_should_enable_debug(void) { + const char *tests_to_debug; + const char *test_debug; + int should_debug = 0; + + /* If TEST_DEBUG is not set, no debug logging */ + test_debug = test_getenv("TEST_DEBUG", NULL); + if (!test_debug) + return 0; + + /* If TESTS_TO_DEBUG is not set, enable for all tests */ + tests_to_debug = test_getenv("TESTS_TO_DEBUG", NULL); + if (!tests_to_debug) + return 1; + + /* Extract test number from current test name (e.g., "0172" from + * "0172-share_consumer_acknowledge") */ + if (!test_curr || !test_curr->name) + return 1; /* Enable by default if test name not available */ + + /* Simple comma-separated parsing */ + const char *p = tests_to_debug; + while (*p) { + const char *comma; + size_t token_len; + + /* Skip leading whitespace */ + while (*p && isspace((unsigned char)*p)) + p++; + + if (!*p) + break; + + /* Find next comma or end of string */ + comma = strchr(p, ','); + if (comma) + token_len = (size_t)(comma - p); + else + token_len = strlen(p); + + /* Trim trailing whitespace */ + while (token_len > 0 && + isspace((unsigned char)p[token_len - 1])) + token_len--; + + /* Check if current test name starts with this token */ + if (token_len > 0 && + strncmp(test_curr->name, p, token_len) == 0) { + should_debug = 1; + break; + } + + /* Move to next token */ + if (comma) + p = comma + 1; + else + break; + } + + return should_debug; +} + void test_conf_common_init(rd_kafka_conf_t *conf, int timeout) { if (conf) { const char *tmp = test_getenv("TEST_DEBUG", NULL); - if (tmp) + if (tmp && test_should_enable_debug()) test_conf_set(conf, "debug", tmp); } @@ -1506,9 +1668,10 @@ static void run_tests(int argc, char **argv) { } else if (tests_to_skip && strstr(tests_to_skip, testnum)) skip_reason = "included in TESTS_SKIP list"; else if (skip_tests_till) { - if (!strcmp(skip_tests_till, testnum)) + if (!strcmp(skip_tests_till, testnum)) { + rd_free(skip_tests_till); skip_tests_till = NULL; - else + } else skip_reason = "ignoring test before TESTS_SKIP_BEFORE"; } @@ -1814,8 +1977,15 @@ static void test_cleanup(void) { test->report_arr = NULL; } - if (test_sql_cmd) - rd_free(test_sql_cmd); + RD_IF_FREE(test_sql_cmd, rd_free); + RD_IF_FREE(test_consumer_group_protocol_str, rd_free); + RD_IF_FREE(tests_to_run, rd_free); + RD_IF_FREE(subtests_to_run, rd_free); + RD_IF_FREE(tests_to_skip, rd_free); + RD_IF_FREE(skip_tests_till, rd_free); + RD_IF_FREE(test_broker_version_str, rd_free); + RD_IF_FREE(test_git_version, rd_free); + RD_IF_FREE(test_sockem_conf, rd_free); } @@ -1833,17 +2003,23 @@ int main(int argc, char **argv) { #ifndef _WIN32 signal(SIGINT, test_sig_term); #endif - tests_to_run = test_getenv("TESTS", NULL); - subtests_to_run = test_getenv("SUBTESTS", NULL); - tests_to_skip = test_getenv("TESTS_SKIP", NULL); - tmpver = test_getenv("TEST_KAFKA_VERSION", NULL); - skip_tests_till = test_getenv("TESTS_SKIP_BEFORE", NULL); - + tests_to_run = (char *)test_getenv("TESTS", NULL); + if (tests_to_run) + tests_to_run = rd_strdup(tests_to_run); + subtests_to_run = (char *)test_getenv("SUBTESTS", NULL); + if (subtests_to_run) + subtests_to_run = rd_strdup(subtests_to_run); + tests_to_skip = (char *)test_getenv("TESTS_SKIP", NULL); + if (tests_to_skip) + tests_to_skip = rd_strdup(tests_to_skip); + skip_tests_till = (char *)test_getenv("TESTS_SKIP_BEFORE", NULL); + if (skip_tests_till) + skip_tests_till = rd_strdup(skip_tests_till); + tmpver = test_getenv("TEST_KAFKA_VERSION", NULL); if (!tmpver) tmpver = test_getenv("KAFKA_VERSION", test_broker_version_str); - test_broker_version_str = tmpver; - - test_git_version = test_getenv("RDKAFKA_GITVER", "HEAD"); + test_broker_version_str = rd_strdup(tmpver); + test_git_version = rd_strdup(test_getenv("RDKAFKA_GITVER", "HEAD")); /* Are we running on CI? */ if (test_getenv("CI", NULL)) { @@ -2718,6 +2894,30 @@ rd_kafka_t *test_create_consumer( return rk; } + +/** + * @brief Produce messages to a topic using a specific producer handle. + * + * This is a convenience wrapper that creates a producer topic, produces + * messages, and cleans up. Commonly used with a shared producer instance + * across multiple test cases. + * + * @param rk Producer handle to use. + * @param topic Topic name. + * @param partition Partition to produce to. + * @param msgcnt Number of messages to produce. + */ +void test_produce_msgs_simple(rd_kafka_t *rk, + const char *topic, + int32_t partition, + int msgcnt) { + rd_kafka_topic_t *rkt; + + rkt = test_create_producer_topic(rk, topic, NULL); + test_produce_msgs(rk, rkt, 0, partition, 0, msgcnt, NULL, 0); + rd_kafka_topic_destroy(rkt); +} + rd_kafka_topic_t *test_create_consumer_topic(rd_kafka_t *rk, const char *topic) { rd_kafka_topic_t *rkt; @@ -3101,6 +3301,106 @@ void test_consumer_verify_assignment0(const char *func, rd_kafka_topic_partition_list_destroy(assignment); } +/** + * @brief Verify that the consumer's assignment matches the expected assignment. + * passed as a topic partition list in \p expected_assignment . + */ +rd_bool_t test_consumer_verify_assignment_topic_partition_list0( + const char *func, + int line, + rd_kafka_t *rk, + const rd_kafka_topic_partition_list_t *expected_assignment) { + rd_kafka_topic_partition_list_t *assignment, + *expected_assignment_copy = NULL; + rd_kafka_resp_err_t err; + int i; + rd_bool_t ret = rd_true; + + if ((err = rd_kafka_assignment(rk, &assignment))) + TEST_FAIL("%s:%d: Failed to get assignment for %s: %s", func, + line, rd_kafka_name(rk), rd_kafka_err2str(err)); + + TEST_SAYL(4, "%s assignment (%d partition(s)):\n", rd_kafka_name(rk), + assignment->cnt); + for (i = 0; i < assignment->cnt; i++) + TEST_SAYL(4, " %s [%" PRId32 "]\n", assignment->elems[i].topic, + assignment->elems[i].partition); + + if (assignment->cnt != expected_assignment->cnt) { + ret = rd_false; + goto done; + } + + expected_assignment_copy = + rd_kafka_topic_partition_list_copy(expected_assignment); + rd_kafka_topic_partition_list_sort(assignment, NULL, NULL); + rd_kafka_topic_partition_list_sort(expected_assignment_copy, NULL, + NULL); + + for (i = 0; i < assignment->cnt; i++) { + if (strcmp(assignment->elems[i].topic, + expected_assignment_copy->elems[i].topic) || + assignment->elems[i].partition != + expected_assignment_copy->elems[i].partition) { + ret = rd_false; + goto done; + } + } + +done: + RD_IF_FREE(expected_assignment_copy, + rd_kafka_topic_partition_list_destroy); + rd_kafka_topic_partition_list_destroy(assignment); + return ret; +} + +/** + * @brief Wait until the consumer's assignment matches the expected assignment. + * passed as a topic partition list in \p expected_assignment . + * Polling if \p do_poll is true, otherwise sleeps. + * Until \p timeout_ms milliseconds. + */ +void test_consumer_wait_assignment_topic_partition_list0( + const char *func, + int line, + rd_kafka_t *rk, + rd_bool_t do_poll, + const rd_kafka_topic_partition_list_t *expected_assignment, + int timeout_ms) { + int i; + rd_ts_t end = test_clock() + timeout_ms * 1000; + rd_bool_t verified = rd_false; + + TEST_SAY("Verifying assignment\n"); + TEST_SAYL(4, "%s expected assignment (%d partition(s)):\n", + rd_kafka_name(rk), expected_assignment->cnt); + for (i = 0; i < expected_assignment->cnt; i++) + TEST_SAYL(4, " %s [%" PRId32 "]\n", + expected_assignment->elems[i].topic, + expected_assignment->elems[i].partition); + + do { + verified = + test_consumer_verify_assignment_topic_partition_list0( + func, line, rk, expected_assignment); + if (verified) + break; + + if (do_poll) + test_consumer_poll_once(rk, NULL, 100); + else + rd_usleep(100 * 1000, NULL); + + } while (test_clock() < end); + + if (!verified) { + TEST_FAIL( + "%s:%d: Expected assignment not found in %s's " + "assignment within timeout %d", + func, line, rd_kafka_name(rk), timeout_ms); + } + TEST_SAY("Verified assignment\n"); +} /** @@ -3149,7 +3449,6 @@ void test_consumer_subscribe_multi(rd_kafka_t *rk, int topic_count, ...) { rd_kafka_topic_partition_list_destroy(topics); } - void test_consumer_assign(const char *what, rd_kafka_t *rk, rd_kafka_topic_partition_list_t *partitions) { @@ -5209,6 +5508,13 @@ static void test_admin_delete_topic(rd_kafka_t *use_rk, const char *topicname) { * @brief Delete a topic */ void test_delete_topic(rd_kafka_t *use_rk, const char *topicname) { + if (!strcmp(test_getenv("TEST_BROKER_OS", ""), "windows")) { + TEST_SAY( + "Skipping deletion of topic \"%s\": broker on Windows " + "(NTFS file locking causes broker shutdown on rename)\n", + topicname); + return; + } if (test_broker_version < TEST_BRKVER(0, 10, 2, 0)) test_delete_topic_sh(topicname); else @@ -5216,6 +5522,29 @@ void test_delete_topic(rd_kafka_t *use_rk, const char *topicname) { } +/** + * @brief Alter group configurations using a dedicated producer handle. + * Admin client APIs should not reuse consumer handles. + * Works for both share groups and regular consumer groups. + */ +void test_alter_group_configurations(const char *group_name, + const char **cfg, + size_t cfg_cnt) { + rd_kafka_t *admin; + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + admin = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + TEST_ASSERT(admin, "Failed to create admin client: %s", errstr); + + test_IncrementalAlterConfigs_simple(admin, RD_KAFKA_RESOURCE_GROUP, + group_name, cfg, cfg_cnt); + + rd_kafka_destroy(admin); +} + + /** * @brief Create additional partitions for a topic using Admin API */ @@ -7419,7 +7748,7 @@ rd_kafka_mock_cluster_t *test_mock_cluster_new(int broker_cnt, * received by mock cluster \p mcluster, matching * function \p match , called with opaque \p opaque . */ -static size_t test_mock_get_matching_request_cnt( +size_t test_mock_get_matching_request_cnt( rd_kafka_mock_cluster_t *mcluster, rd_bool_t (*match)(rd_kafka_mock_request_t *request, void *opaque), void *opaque) { @@ -7661,3 +7990,387 @@ int test_consumer_group_protocol_classic() { return !test_consumer_group_protocol_str || !strcmp(test_consumer_group_protocol_str, "classic"); } + + +/**************************************************************************** + * Share Consumer helpers + ****************************************************************************/ + +/** + * @brief Get underlying rd_kafka_t handle from share consumer. + * + * @param rkshare Share consumer handle. + * @returns The underlying rd_kafka_t handle. + */ +rd_kafka_t *test_share_consumer_get_rk(rd_kafka_share_t *rkshare) { + return rkshare->rkshare_rk; +} + + +/** + * @brief Create a share consumer with standard configuration. + * + * @param group_id The share group ID. + * + * @returns A new share consumer instance. + * + * @remark TODO: Remove explicit group.id and enable.auto.commit settings + * once these properties are added as defaults to + * rd_kafka_share_consumer_new(). + */ +rd_kafka_share_t *test_create_share_consumer(const char *group_id, + const char *ack_mode) { + rd_kafka_share_t *rk; + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + + rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr)); + + if (ack_mode) { + rd_kafka_conf_set(conf, "share.acknowledgement.mode", ack_mode, + errstr, sizeof(errstr)); + } + + rk = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rk, "Failed to create share consumer: %s", errstr); + + return rk; +} + + +/** + * @brief Consume share messages and verify they come from expected topics. + * + * @param rk Share consumer handle. + * @param timeout_ms Poll timeout in milliseconds. + * @param expected_topics Array of topic names that are valid sources + * (NULL to skip verification). + * @param expected_topic_cnt Number of topics in expected_topics array. + * @param out_valid Output: count of valid (non-error) messages received. + * + * @returns 0 on success, -1 if message from unexpected topic received. + */ +int test_share_poll(rd_kafka_share_t *rk, + int timeout_ms, + const char **expected_topics, + int expected_topic_cnt, + int *out_valid) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *err; + size_t rcvd, i; + int valid = 0; + int j; + int ret = 0; + + err = rd_kafka_share_poll(rk, timeout_ms, &batch); + if (err) { + rd_kafka_error_destroy(err); + rd_kafka_messages_destroy(batch); + *out_valid = 0; + return 0; + } + + rcvd = rd_kafka_messages_count(batch); + for (i = 0; i < rcvd; i++) { + rd_kafka_message_t *msg = rd_kafka_messages_get(batch, i); + + if (msg->err) { + /* Skip error messages (destroyed by + * rd_kafka_messages_destroy() below) */ + continue; + } + + if (expected_topics) { + const char *msg_topic = rd_kafka_topic_name(msg->rkt); + int found = 0; + + /* Verify message is from expected topic */ + for (j = 0; j < expected_topic_cnt; j++) { + if (strcmp(msg_topic, expected_topics[j]) == + 0) { + found = 1; + break; + } + } + + if (!found) { + TEST_SAY( + "ERROR: Received message from " + "unexpected topic '%s'\n", + msg_topic); + ret = -1; + } + } + valid++; + } + + rd_kafka_messages_destroy(batch); + *out_valid = valid; + return ret; +} + + +/** + * @brief Consume share messages until expected count or max attempts. + * + * @param rk Share consumer handle. + * @param expected Number of messages to consume. + * @param max_attempts Maximum poll attempts. + * @param timeout_ms Timeout per poll in milliseconds. + * @param expected_topics Array of valid topic names (NULL to skip + * verification). + * @param expected_topic_cnt Number of topics in expected_topics. + * + * @returns Number of messages consumed, or -1 if message from wrong topic. + */ +int test_share_consume_msgs(rd_kafka_share_t *rk, + int expected, + int max_attempts, + int timeout_ms, + const char **expected_topics, + int expected_topic_cnt) { + int total = 0; + + while (total < expected && max_attempts-- > 0) { + int batch_cnt = 0; + int ret; + + ret = test_share_poll(rk, timeout_ms, expected_topics, + expected_topic_cnt, &batch_cnt); + if (ret < 0) + return -1; /* Wrong topic detected */ + + total += batch_cnt; + } + + return total; +} + + +/** + * @brief Start subscribing for multiple topics (share consumers). + */ +void test_share_consumer_subscribe_multi(rd_kafka_share_t *rk, + int topic_count, + ...) { + rd_kafka_topic_partition_list_t *topics; + rd_kafka_resp_err_t err; + va_list ap; + int i; + + topics = rd_kafka_topic_partition_list_new(topic_count); + + va_start(ap, topic_count); + for (i = 0; i < topic_count; i++) { + const char *topic = va_arg(ap, const char *); + rd_kafka_topic_partition_list_add(topics, topic, + RD_KAFKA_PARTITION_UA); + } + va_end(ap); + + err = rd_kafka_share_subscribe(rk, topics); + if (err) + TEST_FAIL("%s: Failed to subscribe to topics: %s\n", + rd_kafka_name(test_share_consumer_get_rk(rk)), + rd_kafka_err2str(err)); + + rd_kafka_topic_partition_list_destroy(topics); +} + + +/** + * @brief Get current subscription list for share consumers. + * + * @returns The subscription list. Caller must destroy with + * rd_kafka_topic_partition_list_destroy(). + */ +rd_kafka_topic_partition_list_t *test_get_subscription(rd_kafka_share_t *rk) { + rd_kafka_topic_partition_list_t *subscription = NULL; + rd_kafka_resp_err_t err; + + err = rd_kafka_share_subscription(rk, &subscription); + if (err) + TEST_FAIL("%s: Failed to get subscription: %s\n", + rd_kafka_name(test_share_consumer_get_rk(rk)), + rd_kafka_err2str(err)); + + TEST_ASSERT(subscription != NULL, + "%s: subscription() returned NULL list", + rd_kafka_name(test_share_consumer_get_rk(rk))); + + return subscription; +} + + +/** + * @brief Set share.auto.offset.reset configuration for a share group. + * + * @param group_name The share group name. + * @param reset_policy The offset reset policy ("earliest" or "latest"). + */ +void test_share_set_auto_offset_reset(const char *group_name, + const char *reset_policy) { + const char *cfg[] = {"share.auto.offset.reset", "SET", reset_policy}; + test_alter_group_configurations(group_name, cfg, 1); +} + + +/** + * @brief Set share.isolation.level configuration for a share group. + * + * @param group_name The share group name. + * @param isolation_level The isolation level ("read_committed" or + * "read_uncommitted"). + */ +void test_share_set_isolation_level(const char *group_name, + const char *isolation_level) { + const char *cfg[] = {"share.isolation.level", "SET", isolation_level}; + test_alter_group_configurations(group_name, cfg, 1); +} + +/** + * @brief Close a share consumer. + * + * @param rkshare The share consumer to close. + */ +void test_share_consumer_close(rd_kafka_share_t *rkshare) { + rd_kafka_error_t *error; + + TEST_SAY("Calling rd_kafka_share_consumer_close\n"); + error = rd_kafka_share_consumer_close(rkshare); + TEST_SAY("Completed rd_kafka_share_consumer_close\n"); + + if (error) + TEST_FAIL("Failed to close share consumer: %s\n", + rd_kafka_error_string(error)); +} + +/** + * @brief Destroy a share consumer. + * + * @param rkshare The share consumer to destroy. + */ +void test_share_destroy(rd_kafka_share_t *rkshare) { + + TEST_SAY("Calling rd_kafka_share_destroy\n"); + rd_kafka_share_destroy(rkshare); + TEST_SAY("Completed rd_kafka_share_destroy\n"); +} + +/** + * @brief Standard share acknowledgement callback. + * + * Tracks callback invocations, offsets acknowledged, and errors. + * Opaque must be a pointer to test_ack_cb_state_t. + */ +void test_ack_cb_state_push_err(test_ack_cb_state_t *state, + rd_kafka_resp_err_t err) { + state->errs = rd_realloc(state->errs, sizeof(*state->errs) * + (state->callback_cnt + 1)); + state->errs[state->callback_cnt] = err; + state->callback_cnt++; +} + +void test_ack_cb_state_destroy(test_ack_cb_state_t *state) { + if (state->errs) + rd_free(state->errs); + state->errs = NULL; + state->callback_cnt = 0; + state->total_offsets = 0; +} + +rd_kafka_resp_err_t +test_ack_cb_state_first_err(const test_ack_cb_state_t *state) { + int i; + for (i = 0; i < state->callback_cnt; i++) { + if (state->errs[i] != RD_KAFKA_RESP_ERR_NO_ERROR) + return state->errs[i]; + } + return RD_KAFKA_RESP_ERR_NO_ERROR; +} + +void test_share_ack_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque) { + test_ack_cb_state_t *state = (test_ack_cb_state_t *)opaque; + const rd_kafka_share_partition_offsets_t *entry; + + (void)rkshare; + + test_ack_cb_state_push_err(state, err); + + entry = rd_kafka_share_partition_offsets_list_get(partitions, 0); + if (entry) + state->total_offsets += + rd_kafka_share_partition_offsets_offsets_cnt(entry); +} + +/** + * @brief Create share consumer with callback and custom acknowledgement mode. + */ +rd_kafka_share_t *test_create_share_consumer_with_cb( + const char *group_id, + const char *ack_mode, + test_ack_cb_state_t *state, + void (*cb)(rd_kafka_share_t *, + rd_kafka_share_partition_offsets_list_t *, + rd_kafka_resp_err_t, + void *)) { + rd_kafka_share_t *rkshare; + rd_kafka_conf_t *conf; + char errstr[512]; + + test_conf_init(&conf, NULL, 0); + rd_kafka_conf_set(conf, "group.id", group_id, errstr, sizeof(errstr)); + rd_kafka_conf_set(conf, "share.acknowledgement.mode", ack_mode, errstr, + sizeof(errstr)); + + rkshare = rd_kafka_share_consumer_new(conf, errstr, sizeof(errstr)); + TEST_ASSERT(rkshare, "Failed to create share consumer: %s", errstr); + + /* Register acknowledgement callback at runtime */ + rd_kafka_error_t *error = rd_kafka_share_set_acknowledgement_commit_cb( + rkshare, cb ? cb : test_share_ack_cb, state); + TEST_ASSERT(error == NULL, + "Failed to set acknowledgement commit callback: %s", + rd_kafka_error_string(error)); + return rkshare; +} + +/** + * @brief Wait for acknowledgement callbacks while polling. + */ +rd_bool_t test_wait_for_cb_with_poll(test_ack_cb_state_t *state, + rd_kafka_share_t *rkshare, + int min_callbacks, + int timeout_ms) { + rd_bool_t success = rd_false; + int elapsed = 0; + int poll_interval = 100; + + while (elapsed < timeout_ms) { + rd_kafka_messages_t *batch = NULL; + rd_kafka_error_t *error = + rd_kafka_share_poll(rkshare, poll_interval, &batch); + if (error) { + TEST_SAY( + "test_wait_for_cb_with_poll: " + "share_poll err=%s (%s) rcvd=%zu\n", + rd_kafka_err2name(rd_kafka_error_code(error)), + rd_kafka_error_string(error), + rd_kafka_messages_count(batch)); + rd_kafka_error_destroy(error); + } + rd_kafka_messages_destroy(batch); + + if (state->callback_cnt >= min_callbacks) { + success = rd_true; + break; + } + elapsed += poll_interval; + } + return success; +} diff --git a/lib/librdkafka-2.10.1/tests/test.conf.example b/lib/librdkafka-2.15.0/tests/test.conf.example similarity index 100% rename from lib/librdkafka-2.10.1/tests/test.conf.example rename to lib/librdkafka-2.15.0/tests/test.conf.example diff --git a/lib/librdkafka-2.10.1/tests/test.h b/lib/librdkafka-2.15.0/tests/test.h similarity index 87% rename from lib/librdkafka-2.10.1/tests/test.h rename to lib/librdkafka-2.15.0/tests/test.h index 4cecf68d5fe..a797c70a852 100644 --- a/lib/librdkafka-2.10.1/tests/test.h +++ b/lib/librdkafka-2.15.0/tests/test.h @@ -554,6 +554,12 @@ rd_kafka_t *test_create_consumer( void *opaque), rd_kafka_conf_t *conf, rd_kafka_topic_conf_t *default_topic_conf); + +void test_produce_msgs_simple(rd_kafka_t *rk, + const char *topic, + int32_t partition, + int msgcnt); + rd_kafka_topic_t *test_create_consumer_topic(rd_kafka_t *rk, const char *topic); rd_kafka_topic_t * test_create_topic_object(rd_kafka_t *rk, const char *topic, ...); @@ -676,6 +682,28 @@ void test_consumer_verify_assignment0(const char *func, test_consumer_verify_assignment0(__FUNCTION__, __LINE__, rk, \ fail_immediately, __VA_ARGS__) +rd_bool_t test_consumer_verify_assignment_topic_partition_list0( + const char *func, + int line, + rd_kafka_t *rk, + const rd_kafka_topic_partition_list_t *expected_assignment); +#define test_consumer_verify_assignment_topic_partition_list( \ + rk, expected_assignment) \ + test_consumer_verify_assignment_topic_partition_list0( \ + __FUNCTION__, __LINE__, rk, expected_assignment) + +void test_consumer_wait_assignment_topic_partition_list0( + const char *func, + int line, + rd_kafka_t *rk, + rd_bool_t do_poll, + const rd_kafka_topic_partition_list_t *expected_assignment, + int timeout_ms); +#define test_consumer_wait_assignment_topic_partition_list( \ + rk, do_poll, expected_assignment, timeout_ms) \ + test_consumer_wait_assignment_topic_partition_list0( \ + __FUNCTION__, __LINE__, rk, do_poll, expected_assignment, \ + timeout_ms) void test_consumer_assign(const char *what, rd_kafka_t *rk, @@ -778,6 +806,7 @@ void test_prepare_msg(uint64_t testid, #if WITH_SOCKEM void test_socket_enable(rd_kafka_conf_t *conf); +void *test_socket_find(struct test *test, sockem_t *skm); void test_socket_close_all(struct test *test, int reinit); int test_socket_sockem_set_all(const char *key, int val); void test_socket_sockem_set(int s, const char *key, int value); @@ -840,6 +869,10 @@ test_IncrementalAlterConfigs_simple(rd_kafka_t *rk, const char **configs, size_t config_cnt); +void test_alter_group_configurations(const char *group_name, + const char **cfg, + size_t cfg_cnt); + rd_kafka_resp_err_t test_DeleteGroups_simple(rd_kafka_t *rk, rd_kafka_queue_t *useq, char **groups, @@ -877,6 +910,12 @@ rd_kafka_resp_err_t test_delete_all_test_topics(int timeout_ms); void test_mock_cluster_destroy(rd_kafka_mock_cluster_t *mcluster); rd_kafka_mock_cluster_t *test_mock_cluster_new(int broker_cnt, const char **bootstraps); + +size_t test_mock_get_matching_request_cnt( + rd_kafka_mock_cluster_t *mcluster, + rd_bool_t (*match)(rd_kafka_mock_request_t *request, void *opaque), + void *opaque); + size_t test_mock_wait_matching_requests( rd_kafka_mock_cluster_t *mcluster, size_t num, @@ -987,6 +1026,118 @@ int test_consumer_group_protocol_classic(); rd_kafka_error_name(_e), rd_kafka_error_string(_e)); \ } while (0) +/**************************************************************************** + * Share Consumer helpers + ****************************************************************************/ + +rd_kafka_t *test_share_consumer_get_rk(rd_kafka_share_t *rkshare); + +rd_kafka_share_t *test_create_share_consumer(const char *group_id, + const char *ack_mode); + +int test_share_poll(rd_kafka_share_t *rk, + int timeout_ms, + const char **expected_topics, + int expected_topic_cnt, + int *out_valid); + +int test_share_consume_msgs(rd_kafka_share_t *rk, + int expected, + int max_attempts, + int timeout_ms, + const char **expected_topics, + int expected_topic_cnt); + +void test_share_consumer_subscribe_multi(rd_kafka_share_t *rk, + int topic_count, + ...); + +rd_kafka_topic_partition_list_t *test_get_subscription(rd_kafka_share_t *rk); + +void test_share_set_auto_offset_reset(const char *group_name, + const char *reset_policy); + +void test_share_set_isolation_level(const char *group_name, + const char *isolation_level); + +void test_share_consumer_close(rd_kafka_share_t *rkshare); +void test_share_destroy(rd_kafka_share_t *rkshare); + +/** + * @brief State for tracking share acknowledgement callbacks. + * + * @remark Call test_ack_cb_state_destroy() to free the @c errs allocation. + */ +typedef struct test_ack_cb_state_s { + int callback_cnt; /**< Number of callbacks invoked */ + size_t total_offsets; /**< Total offsets acknowledged */ + rd_kafka_resp_err_t *errs; /**< Per-callback err, length callback_cnt */ +} test_ack_cb_state_t; + +/** + * @brief Record the err for one callback invocation. Grows @c errs by one. + */ +void test_ack_cb_state_push_err(test_ack_cb_state_t *state, + rd_kafka_resp_err_t err); + +/** + * @brief Free the @c errs allocation and zero counters. Safe to call on a + * zero-initialized state. + */ +void test_ack_cb_state_destroy(test_ack_cb_state_t *state); + +/** + * @brief Returns the first non-NO_ERROR err recorded, or NO_ERROR if all + * callbacks were clean (or none have fired yet). + */ +rd_kafka_resp_err_t +test_ack_cb_state_first_err(const test_ack_cb_state_t *state); + +/** + * @brief Standard share acknowledgement callback. + * + * Tracks callback invocations, offsets acknowledged, and errors. + * Opaque must be a pointer to test_ack_cb_state_t. + */ +void test_share_ack_cb(rd_kafka_share_t *rkshare, + rd_kafka_share_partition_offsets_list_t *partitions, + rd_kafka_resp_err_t err, + void *opaque); + +/** + * @brief Create share consumer with callback and custom acknowledgement mode. + * + * @param group_id Share group ID + * @param ack_mode Acknowledgement mode ("implicit" or "explicit") + * @param state Callback state (will be set as opaque) + * @param cb Custom callback function (NULL to use test_share_ack_cb) + * @returns Created share consumer + */ +rd_kafka_share_t *test_create_share_consumer_with_cb( + const char *group_id, + const char *ack_mode, + test_ack_cb_state_t *state, + void (*cb)(rd_kafka_share_t *, + rd_kafka_share_partition_offsets_list_t *, + rd_kafka_resp_err_t, + void *)); + +/** + * @brief Wait for acknowledgement callbacks while polling. + * + * Polls for messages and waits until at least min_callbacks have been invoked. + * + * @param state Callback state to monitor + * @param rkshare Share consumer to poll + * @param min_callbacks Minimum number of callbacks to wait for + * @param timeout_ms Maximum time to wait in milliseconds + * @returns rd_true if min_callbacks reached, rd_false on timeout + */ +rd_bool_t test_wait_for_cb_with_poll(test_ack_cb_state_t *state, + rd_kafka_share_t *rkshare, + int min_callbacks, + int timeout_ms); + /** * @name rusage.c * @{ diff --git a/lib/librdkafka-2.10.1/tests/testcpp.cpp b/lib/librdkafka-2.15.0/tests/testcpp.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/testcpp.cpp rename to lib/librdkafka-2.15.0/tests/testcpp.cpp diff --git a/lib/librdkafka-2.10.1/tests/testcpp.h b/lib/librdkafka-2.15.0/tests/testcpp.h similarity index 100% rename from lib/librdkafka-2.10.1/tests/testcpp.h rename to lib/librdkafka-2.15.0/tests/testcpp.h diff --git a/lib/librdkafka-2.10.1/tests/testshared.h b/lib/librdkafka-2.15.0/tests/testshared.h similarity index 100% rename from lib/librdkafka-2.10.1/tests/testshared.h rename to lib/librdkafka-2.15.0/tests/testshared.h diff --git a/lib/librdkafka-2.10.1/tests/tools/README.md b/lib/librdkafka-2.15.0/tests/tools/README.md similarity index 100% rename from lib/librdkafka-2.10.1/tests/tools/README.md rename to lib/librdkafka-2.15.0/tests/tools/README.md diff --git a/lib/librdkafka-2.10.1/tests/tools/stats/README.md b/lib/librdkafka-2.15.0/tests/tools/stats/README.md similarity index 100% rename from lib/librdkafka-2.10.1/tests/tools/stats/README.md rename to lib/librdkafka-2.15.0/tests/tools/stats/README.md diff --git a/lib/librdkafka-2.10.1/tests/tools/stats/filter.jq b/lib/librdkafka-2.15.0/tests/tools/stats/filter.jq similarity index 100% rename from lib/librdkafka-2.10.1/tests/tools/stats/filter.jq rename to lib/librdkafka-2.15.0/tests/tools/stats/filter.jq diff --git a/lib/librdkafka-2.10.1/tests/tools/stats/graph.py b/lib/librdkafka-2.15.0/tests/tools/stats/graph.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/tools/stats/graph.py rename to lib/librdkafka-2.15.0/tests/tools/stats/graph.py diff --git a/lib/librdkafka-2.10.1/tests/tools/stats/requirements.txt b/lib/librdkafka-2.15.0/tests/tools/stats/requirements.txt similarity index 100% rename from lib/librdkafka-2.10.1/tests/tools/stats/requirements.txt rename to lib/librdkafka-2.15.0/tests/tools/stats/requirements.txt diff --git a/lib/librdkafka-2.10.1/tests/tools/stats/to_csv.py b/lib/librdkafka-2.15.0/tests/tools/stats/to_csv.py similarity index 100% rename from lib/librdkafka-2.10.1/tests/tools/stats/to_csv.py rename to lib/librdkafka-2.15.0/tests/tools/stats/to_csv.py diff --git a/lib/librdkafka-2.15.0/tests/trivup/trivup-0.15.0.tar.gz b/lib/librdkafka-2.15.0/tests/trivup/trivup-0.15.0.tar.gz new file mode 100644 index 00000000000..740833069fb Binary files /dev/null and b/lib/librdkafka-2.15.0/tests/trivup/trivup-0.15.0.tar.gz differ diff --git a/lib/librdkafka-2.10.1/tests/until-fail.sh b/lib/librdkafka-2.15.0/tests/until-fail.sh similarity index 100% rename from lib/librdkafka-2.10.1/tests/until-fail.sh rename to lib/librdkafka-2.15.0/tests/until-fail.sh diff --git a/lib/librdkafka-2.10.1/tests/xxxx-assign_partition.c b/lib/librdkafka-2.15.0/tests/xxxx-assign_partition.c similarity index 100% rename from lib/librdkafka-2.10.1/tests/xxxx-assign_partition.c rename to lib/librdkafka-2.15.0/tests/xxxx-assign_partition.c diff --git a/lib/librdkafka-2.10.1/tests/xxxx-metadata.cpp b/lib/librdkafka-2.15.0/tests/xxxx-metadata.cpp similarity index 100% rename from lib/librdkafka-2.10.1/tests/xxxx-metadata.cpp rename to lib/librdkafka-2.15.0/tests/xxxx-metadata.cpp diff --git a/lib/librdkafka-2.10.1/vcpkg.json b/lib/librdkafka-2.15.0/vcpkg.json similarity index 66% rename from lib/librdkafka-2.10.1/vcpkg.json rename to lib/librdkafka-2.15.0/vcpkg.json index 13e95353643..40ce260914e 100644 --- a/lib/librdkafka-2.10.1/vcpkg.json +++ b/lib/librdkafka-2.15.0/vcpkg.json @@ -1,22 +1,22 @@ { "name": "librdkafka", - "version": "2.10.1", + "version": "2.15.0", "dependencies": [ { "name": "zstd", - "version>=": "1.5.6#0" + "version>=": "1.5.7#0" }, { "name": "zlib", - "version>=": "1.3.1#0" + "version>=": "1.3.2#0" }, { "name": "openssl", - "version>=": "3.3.2#1" + "version>=": "3.6.2#0" }, { "name": "curl", - "version>=": "8.10.1#0" + "version>=": "8.19.0#0" } ], "builtin-baseline": "56765209ec0e92c58a5fd91aa09c46a16d660026" diff --git a/lib/librdkafka-2.10.1/win32/.gitignore b/lib/librdkafka-2.15.0/win32/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/win32/.gitignore rename to lib/librdkafka-2.15.0/win32/.gitignore diff --git a/lib/librdkafka-2.10.1/win32/README.md b/lib/librdkafka-2.15.0/win32/README.md similarity index 100% rename from lib/librdkafka-2.10.1/win32/README.md rename to lib/librdkafka-2.15.0/win32/README.md diff --git a/lib/librdkafka-2.10.1/win32/build-package.bat b/lib/librdkafka-2.15.0/win32/build-package.bat similarity index 100% rename from lib/librdkafka-2.10.1/win32/build-package.bat rename to lib/librdkafka-2.15.0/win32/build-package.bat diff --git a/lib/librdkafka-2.10.1/win32/build.bat b/lib/librdkafka-2.15.0/win32/build.bat similarity index 100% rename from lib/librdkafka-2.10.1/win32/build.bat rename to lib/librdkafka-2.15.0/win32/build.bat diff --git a/lib/librdkafka-2.10.1/win32/common.vcxproj b/lib/librdkafka-2.15.0/win32/common.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/common.vcxproj rename to lib/librdkafka-2.15.0/win32/common.vcxproj diff --git a/lib/librdkafka-2.10.1/win32/interceptor_test/interceptor_test.vcxproj b/lib/librdkafka-2.15.0/win32/interceptor_test/interceptor_test.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/interceptor_test/interceptor_test.vcxproj rename to lib/librdkafka-2.15.0/win32/interceptor_test/interceptor_test.vcxproj diff --git a/lib/librdkafka-2.10.1/win32/librdkafka.autopkg.template b/lib/librdkafka-2.15.0/win32/librdkafka.autopkg.template similarity index 100% rename from lib/librdkafka-2.10.1/win32/librdkafka.autopkg.template rename to lib/librdkafka-2.15.0/win32/librdkafka.autopkg.template diff --git a/lib/librdkafka-2.10.1/win32/librdkafka.master.testing.targets b/lib/librdkafka-2.15.0/win32/librdkafka.master.testing.targets similarity index 100% rename from lib/librdkafka-2.10.1/win32/librdkafka.master.testing.targets rename to lib/librdkafka-2.15.0/win32/librdkafka.master.testing.targets diff --git a/lib/librdkafka-2.10.1/win32/librdkafka.sln b/lib/librdkafka-2.15.0/win32/librdkafka.sln similarity index 100% rename from lib/librdkafka-2.10.1/win32/librdkafka.sln rename to lib/librdkafka-2.15.0/win32/librdkafka.sln diff --git a/lib/librdkafka-2.10.1/win32/librdkafka.vcxproj b/lib/librdkafka-2.15.0/win32/librdkafka.vcxproj similarity index 95% rename from lib/librdkafka-2.10.1/win32/librdkafka.vcxproj rename to lib/librdkafka-2.15.0/win32/librdkafka.vcxproj index b31f895d628..9f4331cced9 100644 --- a/lib/librdkafka-2.10.1/win32/librdkafka.vcxproj +++ b/lib/librdkafka-2.15.0/win32/librdkafka.vcxproj @@ -119,6 +119,7 @@ + @@ -237,6 +238,8 @@ + + @@ -249,6 +252,9 @@ + + + diff --git a/lib/librdkafka-2.10.1/win32/librdkafkacpp/librdkafkacpp.vcxproj b/lib/librdkafka-2.15.0/win32/librdkafkacpp/librdkafkacpp.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/librdkafkacpp/librdkafkacpp.vcxproj rename to lib/librdkafka-2.15.0/win32/librdkafkacpp/librdkafkacpp.vcxproj diff --git a/lib/librdkafka-2.15.0/win32/msbuild.ps1 b/lib/librdkafka-2.15.0/win32/msbuild.ps1 new file mode 100644 index 00000000000..1f94d9c4324 --- /dev/null +++ b/lib/librdkafka-2.15.0/win32/msbuild.ps1 @@ -0,0 +1,58 @@ +param( + [string]$config='Release', + [string]$platform='x64', + [string]$toolset='v142' +) + +$msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -prerelease -products * -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe) + +echo "Using msbuild $msbuild" + +echo "Cleaning $config $platform $toolset" +& $msbuild win32\librdkafka.sln /p:Configuration=$config /p:Platform=$platform /p:PlatformToolset=$toolset /target:Clean + +echo "Building $config $platform $toolset" +& $msbuild win32\librdkafka.sln /p:Configuration=$config /p:Platform=$platform /p:PlatformToolset=$toolset +if ($LASTEXITCODE -ne 0) { + Write-Error "msbuild failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +cd tests +$env:CI = "true"; +$env:TEST_CONSUMER_GROUP_PROTOCOL = "classic"; +& ..\win32\outdir\$toolset\$platform\Release\tests.exe -l -Q +if ($LASTEXITCODE -ne 0) { + Write-Error "Classic group test failed with exit code $LASTEXITCODE" + cd .. + exit $LASTEXITCODE +} + +# Skip tests needing special limits +$TESTS_WITH_INCREASED_NLIMIT = "0153" +$env:TESTS_SKIP = $TESTS_WITH_INCREASED_NLIMIT; +$env:TEST_CONSUMER_GROUP_PROTOCOL = "consumer"; +& ..\win32\outdir\$toolset\$platform\Release\tests.exe -l -Q +if ($LASTEXITCODE -ne 0) { + Write-Error "Consumer group test failed with exit code $LASTEXITCODE" + cd .. + exit $LASTEXITCODE +} +# Now run only those tests with different limits + +# Tests needing increased number of file descriptors +$msvcrtCheck = Add-Type -Member '[DllImport("msvcrt.dll")] public static extern int _getmaxstdio();' -Name "CheckValue" -PassThru +$msvcrtSet = Add-Type -Member '[DllImport("msvcrt.dll")] public static extern int _setmaxstdio(int n);' -Name "SetValue" -PassThru +$originalLimit = $msvcrtCheck::_getmaxstdio() +$msvcrtSet::_setmaxstdio(2048) +$env:TESTS_SKIP = ""; +$env:TESTS = $TESTS_WITH_INCREASED_NLIMIT; +& ..\win32\outdir\$toolset\$platform\Release\tests.exe -l -Q +if ($LASTEXITCODE -ne 0) { + Write-Error "Consumer group test 0153 failed with exit code $LASTEXITCODE" + cd .. + exit $LASTEXITCODE +} +$msvcrtSet::_setmaxstdio($originalLimit) + +cd .. diff --git a/lib/librdkafka-2.10.1/win32/openssl_engine_example/openssl_engine_example.vcxproj b/lib/librdkafka-2.15.0/win32/openssl_engine_example/openssl_engine_example.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/openssl_engine_example/openssl_engine_example.vcxproj rename to lib/librdkafka-2.15.0/win32/openssl_engine_example/openssl_engine_example.vcxproj diff --git a/lib/librdkafka-2.10.1/win32/package-zip.ps1 b/lib/librdkafka-2.15.0/win32/package-zip.ps1 similarity index 93% rename from lib/librdkafka-2.10.1/win32/package-zip.ps1 rename to lib/librdkafka-2.15.0/win32/package-zip.ps1 index 34dd0ab1aa6..079d051c189 100644 --- a/lib/librdkafka-2.10.1/win32/package-zip.ps1 +++ b/lib/librdkafka-2.15.0/win32/package-zip.ps1 @@ -39,7 +39,7 @@ if ("x64" -eq $platform) { Copy-Item "${srcdir}\librdkafka.dll","${srcdir}\librdkafkacpp.dll", "${srcdir}\libcrypto-3${platformpart}.dll","${srcdir}\libssl-3${platformpart}.dll", -"${srcdir}\zlib1.dll","${srcdir}\zstd.dll","${srcdir}\libcurl.dll" -Destination $bindir +"${srcdir}\z.dll","${srcdir}\zstd.dll","${srcdir}\libcurl.dll" -Destination $bindir Copy-Item "${srcdir}\librdkafka.lib","${srcdir}\librdkafkacpp.lib" -Destination $libdir diff --git a/lib/librdkafka-2.10.1/win32/packages/repositories.config b/lib/librdkafka-2.15.0/win32/packages/repositories.config similarity index 100% rename from lib/librdkafka-2.10.1/win32/packages/repositories.config rename to lib/librdkafka-2.15.0/win32/packages/repositories.config diff --git a/lib/librdkafka-2.10.1/win32/push-package.bat b/lib/librdkafka-2.15.0/win32/push-package.bat similarity index 100% rename from lib/librdkafka-2.10.1/win32/push-package.bat rename to lib/librdkafka-2.15.0/win32/push-package.bat diff --git a/lib/librdkafka-2.10.1/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj b/lib/librdkafka-2.15.0/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj rename to lib/librdkafka-2.15.0/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj diff --git a/lib/librdkafka-2.10.1/win32/rdkafka_example/rdkafka_example.vcxproj b/lib/librdkafka-2.15.0/win32/rdkafka_example/rdkafka_example.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/rdkafka_example/rdkafka_example.vcxproj rename to lib/librdkafka-2.15.0/win32/rdkafka_example/rdkafka_example.vcxproj diff --git a/lib/librdkafka-2.10.1/win32/rdkafka_performance/rdkafka_performance.vcxproj b/lib/librdkafka-2.15.0/win32/rdkafka_performance/rdkafka_performance.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/rdkafka_performance/rdkafka_performance.vcxproj rename to lib/librdkafka-2.15.0/win32/rdkafka_performance/rdkafka_performance.vcxproj diff --git a/lib/librdkafka-2.15.0/win32/setup-msys2.ps1 b/lib/librdkafka-2.15.0/win32/setup-msys2.ps1 new file mode 100644 index 00000000000..b832d3494d7 --- /dev/null +++ b/lib/librdkafka-2.15.0/win32/setup-msys2.ps1 @@ -0,0 +1,47 @@ +# Install (if necessary) and set up msys2. +$ErrorActionPreference = "Stop" + +$url="https://github.com/msys2/msys2-installer/releases/download/2025-12-13/msys2-base-x86_64-20251213.sfx.exe" +$sha256="99f2fee9a7b1c344600ac97347e7be23a1f802d8d843b339ec7473a8ed8d49a6" +$cacheKey = "msys2-$sha256-$Env:CACHE_TAG" +$librdkafkaPath = Get-Location; + +try { + cache restore $cacheKey +} catch { + echo "cache command not found" +} +if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) { + echo "Downloading and installing msys2 to c:\msys64" + + (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe') + + # Verify checksum + (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256 + + # Install msys2 + .\msys2-installer.exe -y -oc:\ + + Remove-Item msys2-installer.exe + + # Set up msys2 the first time + echo "Setting up msys" + c:\msys64\usr\bin\bash -lc ' ' + + # Update packages + echo "Updating msys2 packages" + c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'" + # Update pacman" + c:\msys64\usr\bin\bash -lc "pacman -Sy --noconfirm pacman" + # Install needed packages" + c:\msys64\usr\bin\bash -lc "pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc mingw-w64-x86_64-make mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-lz4 mingw-w64-x86_64-zstd" + + try { + powershell -command "cd $librdkafkaPath; cache store $cacheKey c:\msys64" + } catch { + echo "cache command not found" + } + +} else { + echo "Using previously installed msys2" +} diff --git a/lib/librdkafka-2.15.0/win32/setup-vcpkg.ps1 b/lib/librdkafka-2.15.0/win32/setup-vcpkg.ps1 new file mode 100644 index 00000000000..be02e24045f --- /dev/null +++ b/lib/librdkafka-2.15.0/win32/setup-vcpkg.ps1 @@ -0,0 +1,34 @@ +# Set up vcpkg and install required packages. +$version = "2026.04.27" +$vpkgHash=(Get-FileHash ".\librdkafka\vcpkg.json").Hash +$cacheKey = "vcpkg-$version-$Env:triplet-$vpkgHash-$Env:CACHE_TAG" +$librdkafkaPath = ".\librdkafka"; + +try { + cache restore $cacheKey +} catch { + echo "cache command not found" +} +if (!(Test-Path -Path vcpkg/.git)) { + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + git checkout $version + .\bootstrap-vcpkg.bat + cd .. + cd librdkafka + ..\vcpkg\vcpkg integrate install + # Install required packages. + ..\vcpkg\vcpkg --feature-flags=versions install --triplet $Env:triplet + cd .. + try { + cache store $cacheKey .\vcpkg + } catch { + echo "cache command not found" + } +} else { + cd librdkafka + ..\vcpkg\vcpkg integrate install + cd .. + echo "Using previously installed vcpkg" +} + diff --git a/lib/librdkafka-2.10.1/win32/tests/.gitignore b/lib/librdkafka-2.15.0/win32/tests/.gitignore similarity index 100% rename from lib/librdkafka-2.10.1/win32/tests/.gitignore rename to lib/librdkafka-2.15.0/win32/tests/.gitignore diff --git a/lib/librdkafka-2.10.1/win32/tests/test.conf.example b/lib/librdkafka-2.15.0/win32/tests/test.conf.example similarity index 100% rename from lib/librdkafka-2.10.1/win32/tests/test.conf.example rename to lib/librdkafka-2.15.0/win32/tests/test.conf.example diff --git a/lib/librdkafka-2.10.1/win32/tests/tests.vcxproj b/lib/librdkafka-2.15.0/win32/tests/tests.vcxproj similarity index 87% rename from lib/librdkafka-2.10.1/win32/tests/tests.vcxproj rename to lib/librdkafka-2.15.0/win32/tests/tests.vcxproj index 8853806fd16..ddff9d7ebcf 100644 --- a/lib/librdkafka-2.10.1/win32/tests/tests.vcxproj +++ b/lib/librdkafka-2.15.0/win32/tests/tests.vcxproj @@ -10,6 +10,9 @@ Application + + true + @@ -227,10 +230,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/librdkafka-2.10.1/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj b/lib/librdkafka-2.15.0/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj similarity index 100% rename from lib/librdkafka-2.10.1/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj rename to lib/librdkafka-2.15.0/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj diff --git a/lib/librdkafka-2.10.1/win32/wingetopt.c b/lib/librdkafka-2.15.0/win32/wingetopt.c similarity index 100% rename from lib/librdkafka-2.10.1/win32/wingetopt.c rename to lib/librdkafka-2.15.0/win32/wingetopt.c diff --git a/lib/librdkafka-2.10.1/win32/wingetopt.h b/lib/librdkafka-2.15.0/win32/wingetopt.h similarity index 100% rename from lib/librdkafka-2.10.1/win32/wingetopt.h rename to lib/librdkafka-2.15.0/win32/wingetopt.h diff --git a/lib/librdkafka-2.10.1/win32/wintime.h b/lib/librdkafka-2.15.0/win32/wintime.h similarity index 100% rename from lib/librdkafka-2.10.1/win32/wintime.h rename to lib/librdkafka-2.15.0/win32/wintime.h