Enhance Kafka integration tests to support multiple Kafka versions

Signed-off-by: zzzk1 <[email protected]>

.github/workflows/ci-e2e-kafka.yml

@@ -17,8 +17,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jaeger-version: [v1, v2]  # Adjust if there are specific versions of Jaeger
-    name: kafka ${{ matrix.jaeger-version }}
+        jaeger-version: [v1, v2]
+        version:
+          - major: 3.x
+            distribution: kafka
+          - major: 2.x
+            distribution: kafka
+    name: ${{ matrix.version.distribution}} ${{matrix.version.major }} ${{ matrix.jaeger-version }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -31,12 +36,12 @@ jobs:
       with:
         go-version: 1.23.x
 
-    - name: Run Kafka integration tests
+    - name: Run ${{ matrix.version.distribution }} integration tests
       id: test-execution
-      run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }}
+      run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.version.major }}
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:
         files: cover.out
-        flags: kafka-${{ matrix.jaeger-version }}
+        flags: ${{ matrix.version.distribution }}-${{ matrix.version.major }}-${{ matrix.version.jaeger }}

docker-compose/kafka-integration-test/v2/docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  zookeeper:
+    image: bitnami/zookeeper:3.8.4
+    ports:
+      - "2181:2181"
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+
+  kafka:
+    image: bitnami/kafka:2.8.0
+    ports:
+      - "9092:9092"
+    environment:
+      - KAFKA_BROKER_ID=1
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    depends_on:
+      - zookeeper

docker-compose/kafka/docker-compose.yml

@@ -6,7 +6,7 @@ services:
     environment:
       - ALLOW_ANONYMOUS_LOGIN=yes
   kafka:
-    image: index.docker.io/bitnami/kafka@3.7.0
+    image: bitnami/kafka:3.7.0
     ports:
       - '9092:9092'
     environment:
@@ -108,4 +108,4 @@ services:
       jaeger-remote-storage:
         condition: service_healthy
     links:
-      - jaeger-remote-storage
+      - jaeger-remote-storage

scripts/kafka-integration-test.sh

@@ -5,36 +5,49 @@
 
 set -euf -o pipefail
 
-compose_file="docker-compose/kafka-integration-test/docker-compose.yml"
+compose_file="docker-compose/kafka-integration-test/v3/docker-compose.yml"
 jaeger_version=""
+kafka_version=""
 manage_kafka="true"
 success="false"
 
-print_help() {
-  echo "Usage: $0 [-K] -j <jaeger_version>"
+usage() {
+  echo "Usage: $0 [-K] -j <jaeger_version> -v <kafka_version>"
   echo "  -K: do not start or stop Kafka container (useful for local testing)"
   echo "  -j: major version of Jaeger to test (v1|v2)"
+  echo "  -v: kafka major version, e.g: 3.9"
   exit 1
 }
 
+check_arg() {
+  if [ ! $# -eq 3 ]; then
+      echo "ERROR: need exactly three arguments"
+      usage
+    fi
+}
+
 parse_args() {
-  while getopts "j:Kh" opt; do
+  while getopts "j:v:Kh" opt; do
     case "${opt}" in
     j)
       jaeger_version=${OPTARG}
       ;;
+    v)
+      kafka_version=${OPTARG}
+      ;;
     K)
       manage_kafka="false"
       ;;
     *)
-      print_help
+      usage
       ;;
     esac
   done
   if [ "$jaeger_version" != "v1" ] && [ "$jaeger_version" != "v2" ]; then
     echo "Error: Invalid Jaeger version. Valid options are v1 or v2"
-    print_help
+    usage
   fi
+  compose_file="docker-compose/kafka-integration-test/v${kafka_version%%.*}/docker-compose.yml"
 }
 
 setup_kafka() {
@@ -89,7 +102,7 @@ run_integration_test() {
     make jaeger-v2-storage-integration-test
   else
     echo "Unknown Jaeger version ${jaeger_version}."
-    print_help
+    usage
   fi
 }