AKCORE Merge Upstream changes

.github/workflows/docker_scan.yml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Docker Image CVE Scanner
+on:
+  schedule:
+    # This job will run at 3:30 UTC daily
+    - cron: '30 3 * * *'
+  workflow_dispatch:
+jobs:
+  scan_jvm:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # This is an array of supported tags. Make sure this array only contains the supported tags
+        supported_image_tag: ['latest', '3.7.0']
+    steps:
+      - name: Run CVE scan
+        uses: aquasecurity/trivy-action@master
+        if: always()
+        with:
+          image-ref: apache/kafka:${{ matrix.supported_image_tag }}
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+          output: scan_report_jvm_${{ matrix.supported_image_tag }}.txt
+          exit-code: '1'
+      - name: Upload CVE scan report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: scan_report_jvm_${{ matrix.supported_image_tag }}.txt
+          path: scan_report_jvm_${{ matrix.supported_image_tag }}.txt

.gitignore

@@ -34,7 +34,6 @@ Vagrantfile.local
 
 config/server-*
 config/zookeeper-*
-core/data/*
 gradle/wrapper/*.jar
 gradlew.bat
 

Jenkinsfile

@@ -58,6 +58,7 @@ def job = {
             "3.4": "7.4.x",
             "3.5": "7.5.x",
             "3.6": "7.6.x",
+            "3.7": "7.7.x",
             "trunk": "master",
             "master": "master"
     ]

LICENSE-binary

@@ -216,17 +216,17 @@ commons-lang3-3.12.0
 commons-logging-1.2
 commons-validator-1.7
 error_prone_annotations-2.10.0
-jackson-annotations-2.16.0
-jackson-core-2.16.0
-jackson-databind-2.16.0
-jackson-dataformat-csv-2.16.0
-jackson-datatype-jdk8-2.16.0
-jackson-jaxrs-base-2.16.0
-jackson-jaxrs-json-provider-2.16.0
-jackson-module-afterburner-2.16.0
-jackson-module-jaxb-annotations-2.16.0
-jackson-module-scala_2.13-2.16.0
-jackson-module-scala_2.12-2.16.0
+jackson-annotations-2.16.1
+jackson-core-2.16.1
+jackson-databind-2.16.1
+jackson-dataformat-csv-2.16.1
+jackson-datatype-jdk8-2.16.1
+jackson-jaxrs-base-2.16.1
+jackson-jaxrs-json-provider-2.16.1
+jackson-module-afterburner-2.16.1
+jackson-module-jaxb-annotations-2.16.1
+jackson-module-scala_2.13-2.16.1
+jackson-module-scala_2.12-2.16.1
 jakarta.validation-api-2.0.2
 javassist-3.29.2-GA
 jetty-client-9.4.53.v20231009
@@ -258,15 +258,20 @@ plexus-utils-3.5.1
 reflections-0.10.2
 reload4j-1.2.25
 rocksdbjni-7.9.2
+scala-collection-compat_2.12-2.10.0
 scala-collection-compat_2.13-2.10.0
+scala-library-2.12.18
 scala-library-2.13.12
+scala-logging_2.12-3.9.4
 scala-logging_2.13-3.9.4
+scala-reflect-2.12.18
 scala-reflect-2.13.12
+scala-java8-compat_2.12-1.0.2
 scala-java8-compat_2.13-1.0.2
 snappy-java-1.1.10.5
 swagger-annotations-2.2.8
-zookeeper-3.8.3
-zookeeper-jute-3.8.3
+zookeeper-3.8.4
+zookeeper-jute-3.8.4
 
 ===============================================================================
 This product bundles various third-party components under other open source
@@ -328,7 +333,7 @@ zstd-jni-1.5.5-11 see: licenses/zstd-jni-BSD-2-clause
 ---------------------------------------
 BSD 3-Clause
 
-jline-3.22.0, see: licenses/jline-BSD-3-clause
+jline-3.25.1, see: licenses/jline-BSD-3-clause
 jsr305-3.0.2, see: licenses/jsr305-BSD-3-clause
 paranamer-2.8, see: licenses/paranamer-BSD-3-clause
 protobuf-java-3.23.4, see: licenses/protobuf-java-BSD-3-clause

README.md

@@ -54,9 +54,14 @@ Follow instructions in https://kafka.apache.org/quickstart
     ./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
 
 ### Running a particular unit/integration test with log4j output ###
-Change the log4j setting in either `clients/src/test/resources/log4j.properties` or `core/src/test/resources/log4j.properties`
+By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j.properties` file in the module's `src/test/resources` directory.
 
-    ./gradlew clients:test --tests RequestResponseTest
+For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j.properties#L21) in `clients/src/test/resources/log4j.properties` 
+to `log4j.logger.org.apache.kafka=INFO` and then run:
+
+    ./gradlew cleanTest clients:test --tests NetworkClientTest   
+
+And you should see `INFO` level logs in the file under the `clients/build/test-results/test` directory.
 
 ### Specifying test retries ###
 By default, each failed test is retried once up to a maximum of five retries per test run. Tests are retried at the end of the test task. Adjust these parameters in the following way:
@@ -87,15 +92,25 @@ fail due to code changes. You can just run:
 
 ### Running a Kafka broker in KRaft mode
 
+Using compiled files:
+
     KAFKA_CLUSTER_ID="$(./bin/kafka-storage.sh random-uuid)"
     ./bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c config/kraft/server.properties
     ./bin/kafka-server-start.sh config/kraft/server.properties
 
+Using docker image:
+
+    docker run -p 9092:9092 apache/kafka:3.7.0
+
 ### Running a Kafka broker in ZooKeeper mode
 
+Using compiled files:
+
     ./bin/zookeeper-server-start.sh config/zookeeper.properties
     ./bin/kafka-server-start.sh config/server.properties
 
+>Since ZooKeeper mode is already deprecated and planned to be removed in Apache Kafka 4.0, the docker image only supports running in KRaft mode
+
 ### Cleaning the build ###
     ./gradlew clean
 

Vagrantfile

@@ -164,6 +164,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         'role' => 'ce-kafka',
         'Owner' => 'ce-kafka',
         'JenkinsBuildUrl' => ENV['BUILD_URL'],
+        'SemaphoreWorkflowUrl' => ENV['SEMAPHORE_WORKFLOW_URL'],
+        'SemaphoreJobId' => ENV['SEMAPHORE_JOB_ID'],
         'cflt_environment' => 'devel',
         'cflt_partition' => 'onprem',
         'cflt_managed_by' => 'iac',

bin/kafka-consumer-groups.sh

@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-exec $(dirname $0)/kafka-run-class.sh kafka.admin.ConsumerGroupCommand "$@"
+exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.consumer.group.ConsumerGroupCommand "$@"

bin/windows/kafka-consumer-groups.bat

@@ -14,4 +14,4 @@ rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 rem See the License for the specific language governing permissions and
 rem limitations under the License.
 
-"%~dp0kafka-run-class.bat" kafka.admin.ConsumerGroupCommand %*
+"%~dp0kafka-run-class.bat" org.apache.kafka.tools.consumer.group.ConsumerGroupCommand %*