diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml index 59efd2d30..d0c13e364 100644 --- a/.github/workflows/fvt.yml +++ b/.github/workflows/fvt.yml @@ -16,10 +16,18 @@ jobs: matrix: go-version: [1.21.x] kafka-version: [3.3.2, 3.4.1, 3.5.1] + include: + - kafka-version: 3.3.2 + scala-version: 2.13 + - kafka-version: 3.4.1 + scala-version: 2.13 + - kafka-version: 3.5.1 + scala-version: 2.13 env: DEBUG: true GOFLAGS: -trimpath KAFKA_VERSION: ${{ matrix.kafka-version }} + SCALA_VERSION: ${{ matrix.scala-version }} steps: - uses: actions/checkout@v3 - name: Setup Docker diff --git a/Dockerfile.kafka b/Dockerfile.kafka index 2e02efb29..b11d899b9 100644 --- a/Dockerfile.kafka +++ b/Dockerfile.kafka @@ -33,6 +33,11 @@ RUN cd /tmp \ && for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \ && rm -f jaxb-api-2.3.0.jar +WORKDIR /opt/kafka-${KAFKA_VERSION} + +ENV JAVA_MAJOR_VERSION=11 + +RUN sed -e "s/JAVA_MAJOR_VERSION=.*/JAVA_MAJOR_VERSION=${JAVA_MAJOR_VERSION}/" -i"" ./bin/kafka-run-class.sh COPY entrypoint.sh / diff --git a/docker-compose.yml b/docker-compose.yml index 2f852e422..4fdf1a517 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ version: '3.9' services: zookeeper-1: + hostname: 'zookeeper-1' image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: @@ -12,6 +13,7 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-2: + hostname: 'zookeeper-2' image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: @@ -23,6 +25,7 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-3: + hostname: 'zookeeper-3' image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: @@ -34,6 +37,7 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' kafka-1: + hostname: 'kafka-1' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -45,12 +49,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-1:9091', - '--broker', - '1', - '--describe', ] interval: 15s timeout: 15s @@ -78,7 +79,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-2: + hostname: 'kafka-2' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -90,12 +94,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-2:9091', - '--broker', - '2', - '--describe', ] interval: 15s timeout: 15s @@ -123,7 +124,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-3: + hostname: 'kafka-3' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -135,12 +139,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-3:9091', - '--broker', - '3', - '--describe', ] interval: 15s timeout: 15s @@ -168,7 +169,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-4: + hostname: 'kafka-4' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -180,12 +184,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-4:9091', - '--broker', - '4', - '--describe', ] interval: 15s timeout: 15s @@ -213,7 +214,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-5: + hostname: 'kafka-5' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -225,12 +229,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-5:9091', - '--broker', - '5', - '--describe', ] interval: 15s timeout: 15s @@ -258,7 +259,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" toxiproxy: + hostname: 'toxiproxy' image: 'ghcr.io/shopify/toxiproxy:2.4.0' healthcheck: test: ['CMD', '/toxiproxy-cli', 'l'] diff --git a/entrypoint.sh b/entrypoint.sh index d6381c907..7f405f92d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,7 +11,9 @@ fi cd "${KAFKA_HOME}" || exit 1 # discard all empty/commented lines -sed -e '/^#/d' -e '/^$/d' -i"" config/server.properties +sed -e '/^#/d' -e '/^$/d' -i".orig" config/server.properties + +echo "########################################################################" >>config/server.properties # emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka for var in "${!KAFKA_CFG_@}"; do diff --git a/functional_consumer_test.go b/functional_consumer_test.go index 167398b55..9d72c3dd3 100644 --- a/functional_consumer_test.go +++ b/functional_consumer_test.go @@ -46,7 +46,10 @@ func TestConsumerHighWaterMarkOffset(t *testing.T) { setupFunctionalTest(t) defer teardownFunctionalTest(t) - p, err := NewSyncProducer(FunctionalTestEnv.KafkaBrokerAddrs, nil) + config := NewFunctionalTestConfig() + config.Producer.Return.Successes = true + + p, err := NewSyncProducer(FunctionalTestEnv.KafkaBrokerAddrs, config) if err != nil { t.Fatal(err) } @@ -57,7 +60,7 @@ func TestConsumerHighWaterMarkOffset(t *testing.T) { t.Fatal(err) } - c, err := NewConsumer(FunctionalTestEnv.KafkaBrokerAddrs, NewFunctionalTestConfig()) + c, err := NewConsumer(FunctionalTestEnv.KafkaBrokerAddrs, config) if err != nil { t.Fatal(err) } @@ -127,6 +130,7 @@ func TestVersionMatrixLZ4(t *testing.T) { // Support for zstd codec was introduced in v2.1.0.0 func TestVersionMatrixZstd(t *testing.T) { + checkKafkaVersion(t, "2.1.0") metrics.UseNilMetrics = true // disable Sarama's go-metrics library t.Cleanup(func() { metrics.UseNilMetrics = false