test 3.0-action

.github/workflows/build-and-test-on-pr-events.yml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License
+
+name: PR Builds
+
+# Run this workflow every time a new pull request is created in the repository
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  build:
+    # Name the Job
+    name: Build Pull Request and run all unit tests
+    # Set the type of machine to run on
+    runs-on: ubuntu-latest
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          # bring in all history because the gradle versions plugin needs to "walk back" to the closest ancestor tag
+          fetch-depth: 0
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Build with Gradle and run all unit tests
+        # exclude the streams test and connect test
+        # Set maxTestRetries for flaky tests
+        run: ./gradlew -PmaxTestRetries=3 cleanTest rat checkstyleMain checkstyleTest :clients:unitTest :core:unitTest --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed

.github/workflows/build-and-upload-archives-upon-creating-tags.yml

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License
+
+name: Release build on tags
+
+# Run this workflow every time a tag is created/pushed
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    # Name the Job
+    name: Build tagged commit and upload an archive
+    # Set the type of machine to run on
+    runs-on: ubuntu-latest
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          # bring in all history because the gradle versions plugin needs to "walk back" to the closest ancestor tag
+          fetch-depth: 0
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Set up release version env variable
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Print the release version
+        run: |
+          echo "Release version (tag name): ${{ env.RELEASE_VERSION }}"
+      - name: Build with Gradle and run all tests
+        # exclude the streams test and connect test
+        # Set maxTestRetries for flaky tests
+        run: ./gradlew -PmaxTestRetries=3 cleanTest rat checkstyleMain checkstyleTest :clients:test :core:test --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed
+      - name: Upload archive
+        env:
+          JFROG_USERNAME: ${{ secrets.JFROG_USERNAME }}
+          JFROG_API_KEY: ${{ secrets.JFROG_API_KEY }}
+        run: |
+          ./gradlew -Pversion=${{ env.RELEASE_VERSION }} :clients:uploadArchives :core:uploadArchives --no-daemon

.github/workflows/build-on-push-to-release-branch.yml

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License
+
+name: Commit pushed build
+
+# Run this workflow every time a new pull request is created in the repository
+on:
+  push:
+    branches:
+      - 3.0-li
+      - 3.0-li-dev2
+
+jobs:
+  build:
+    # Name the Job
+    name: Build pushed commits with integration tests
+    # Set the type of machine to run on
+    runs-on: ubuntu-latest
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          # bring in all history because the gradle versions plugin needs to "walk back" to the closest ancestor tag
+          fetch-depth: 0
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Build with Gradle and run all unit tests
+        # exclude the streams test and connect test
+        # Run integration tests when pushed to branch
+        # Set maxTestRetries for flaky tests
+        run: ./gradlew -PmaxTestRetries=3 cleanTest rat checkstyleMain checkstyleTest :clients:test :core:test --no-daemon -PxmlFindBugsReport=true -PtestLoggingEvents=started,passed,skipped,failed

README.md

@@ -1,3 +1,59 @@
+testtest LinkedIn Branch of Apache Kafka
+=================
+
+This is the version of Kafka running at LinkedIn.
+
+Kafka was born at LinkedIn. We run thousands of brokers to deliver trillions of
+messages per day.  We run a slightly modified version of Apache Kafka trunk.
+This branch contains the LinkedIn Kafka release.
+
+This branch is made up of:
+
+* Apache Kafka trunk (upstream) up to some branch point, see *-li* branch name for base version, you'll be able to get the exact commit from git 
+* Cherry-picked commits from upstream after branch point
+* Patches that are on their way upstream but we have deployed internally in the meantime
+* Patches that are of no interest to upstream
+
+We are making this branch available for people interested. We will be
+documenting the changes in the near future with some more detailed explanations
+in the [LinkedIn Engineering Blog](https://engineering.linkedin.com/blog).
+
+If you are interested in learning more, we invite you to our [Streaming
+Meetup](https://www.meetup.com/Stream-Processing-Meetup-LinkedIn/) where we
+discuss streaming technologies like [Kafka](http://kafka.apache.org) and
+[Samza](http://samza.apache.org).
+
+You are encouraged to check out other Kafka projects from LinkedIn:
+
+* [Cruise Control](https://github.com/linkedin/cruise-control)
+* [Li-Apache-Kafka-Clients](https://github.com/linkedin/li-apache-kafka-clients)
+* [Burrow](https://github.com/linkedin/Burrow)
+* [Kafka Monitor](https://github.com/linkedin/kafka-monitor)
+
+### CI ###
+We are currently using Github Actions as the CI framework, and the testing results can be found [here](https://github.com/linkedin/kafka/actions).
+To publish a release, go to [the release page](https://github.com/linkedin/kafka/releases) and manually create a new release.
+Once the release tag is created, a test job will be triggered to run the necessary tests. And once the test passes, the artifacts
+will be published to [the bintray hosting LinkedIn projects](https://dl.bintray.com/linkedin/maven/com/linkedin/kafka/kafka_2.12/).
+
+Currently we've configured the CI flow to run only unit tests for 'clients' and 'core' when a pull request is created or updated:
+    ./gradlew :clients:unitTest :core:unitTest
+In contrast, all tests for `clients' and `core' are run when creating a release, which may be significantly longer than running the unit tests:
+    ./gradlew :clients:test :core:test
+The reason for this mixed approach is to get faster feedback from CI during code reviews
+and still gain the more through test coverage when publishing a release.
+
+### Contributing ###
+
+At this moment we are not accepting external contributions directly. Please
+contribute to [Apache Kafka](http://kafka.apache.org).
+
+For security issues with this branch please review
+[LinkedIn Security
+Guidelines](https://www.linkedin.com/help/linkedin/answer/62924/security-vulnerabilities?lang=en).
+General Kafka issues should be communicated via the Kafka community.
+
+
 Apache Kafka
 =================
 See our [web site](https://kafka.apache.org) for details on the project.
@@ -132,20 +188,29 @@ The `eclipse` task has been configured to use `${project_dir}/build_eclipse` as
 build directory (`${project_dir}/bin`) clashes with Kafka's scripts directory and we don't use Gradle's build directory
 to avoid known issues with this configuration.
 
-### Publishing the jar for all version of Scala and for all projects to maven ###
+### Publishing the jar for all projects to maven ###
 The recommended command is:
 
-    ./gradlewAll publish
+    ./gradlew -Pversion=<RELEASE_VERSION_NUMBER> publish
 
 For backwards compatibility, the following also works:
 
-    ./gradlewAll uploadArchives
+    ./gradlew -Pversion=<RELEASE_VERSION_NUMBER> uploadArchives
+
+By default, this command will publish artifacts to a JFrog repository named "kafka" under an account specified by the `JFROG_USERNAME` environment variable; 
+and the `JFROG_API_KEY` environment variable is used for the API key for that account.
+
+If you want to publish for all supported Scala version, change `./gradlew` to `./gradlewAll`.
 
-Please note for this to work you should create/update `${GRADLE_USER_HOME}/gradle.properties` (typically, `~/.gradle/gradle.properties`) and assign the following variables
+If you want to override this to use a different maven repository, you should create/update `${GRADLE_USER_HOME}/gradle.properties` (typically, `~/.gradle/gradle.properties`)
+and assign the following variables
 
     mavenUrl=
     mavenUsername=
     mavenPassword=
+
+Signing is disabled by default. If you need signing, please set the following variables in `gradle.properties` as well:
+
     signing.keyId=
     signing.password=
     signing.secretKeyRingFile=

build.gradle

@@ -114,9 +114,13 @@ ext {
   skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean()
   shouldSign = !skipSigning && !version.endsWith("SNAPSHOT")
 
-  mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : ''
-  mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : ''
-  mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : ''
+  jfrogUsername = System.getenv('JFROG_USERNAME')
+  jfrogApiKey = System.getenv('JFROG_API_KEY')
+
+  // By default, publish to JFrog.
+  mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : jfrogRepoUrl
+  mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : jfrogUsername
+  mavenPassword = project.hasProperty('mavenPassword') ? project.mavenPassword : jfrogApiKey
 
   userShowStandardStreams = project.hasProperty("showStandardStreams") ? showStandardStreams : null
 
@@ -260,8 +264,8 @@ subprojects {
 
             artifactId = archivesBaseName
             pom {
-              name = 'Apache Kafka'
-              url = 'https://kafka.apache.org'
+              name = 'LinkedIn fork of Apache Kafka'
+              url = 'https://github.com/linkedin/kafka'
               licenses {
                 license {
                   name = 'The Apache License, Version 2.0'

checkstyle/import-control.xml

@@ -196,6 +196,7 @@
 
     <subpackage name="utils">
       <allow pkg="org.apache.kafka.common" />
+      <allow class="com.sun.management.HotSpotDiagnosticMXBean" />
     </subpackage>
 
     <subpackage name="quotas">

clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java

@@ -40,6 +40,7 @@
 import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse;
 import org.apache.kafka.common.message.SyncGroupRequestData;
 import org.apache.kafka.common.metrics.Measurable;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -1306,6 +1307,21 @@ public GroupCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
                 this.metricGrpName,
                 "The number of seconds since the last coordinator heartbeat was sent"),
                 lastHeartbeat);
+
+            //HOTFIX - extra liveliness-related metrics
+
+            Measurable lastHeartbeatReceived =
+                new Measurable() {
+                    public double measure(MetricConfig config, long now) {
+                        return TimeUnit.SECONDS.convert(now - heartbeat.lastHeartbeatReceive(), TimeUnit.MILLISECONDS);
+                    }
+                };
+            metrics.addMetric(metrics.metricName("last-heartbeat-received-seconds-ago",
+                this.metricGrpName,
+                "The number of seconds since the last successful controller heartbeat was received"),
+                lastHeartbeatReceived);
+
+            //end HOTFIX
         }
     }
 

clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java

@@ -17,6 +17,7 @@
 package org.apache.kafka.clients.consumer.internals;
 
 import org.apache.kafka.clients.GroupRebalanceConfig;
+import java.util.concurrent.TimeUnit;
 import org.apache.kafka.clients.consumer.CommitFailedException;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
@@ -47,6 +48,7 @@
 import org.apache.kafka.common.message.OffsetCommitRequestData;
 import org.apache.kafka.common.message.OffsetCommitResponseData;
 import org.apache.kafka.common.metrics.Measurable;
+import org.apache.kafka.common.metrics.MetricConfig;
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
@@ -112,6 +114,9 @@ public final class ConsumerCoordinator extends AbstractCoordinator {
     private ConsumerGroupMetadata groupMetadata;
     private final boolean throwOnFetchStableOffsetsUnsupported;
 
+    private volatile long prevPollTime = Long.MIN_VALUE; //volatile for metrics
+
+
     // hold onto request&future for committed offset requests to enable async calls.
     private PendingCommittedOffsetRequest pendingCommittedOffsetRequest = null;
 
@@ -467,6 +472,11 @@ void maybeUpdateSubscriptionMetadata() {
      * @return true iff the operation succeeded
      */
     public boolean poll(Timer timer, boolean waitForJoinGroup) {
+        long currentTime = time.milliseconds();
+        if (prevPollTime > Long.MIN_VALUE) {
+            sensors.pollInterval.record(currentTime - prevPollTime);
+        }
+        prevPollTime = currentTime;
         maybeUpdateSubscriptionMetadata();
 
         invokeCompletedOffsetCommitCallbacks();
@@ -1408,6 +1418,7 @@ private class ConsumerCoordinatorMetrics {
         private final Sensor revokeCallbackSensor;
         private final Sensor assignCallbackSensor;
         private final Sensor loseCallbackSensor;
+        private final Sensor pollInterval;
 
         private ConsumerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
             this.metricGrpName = metricGrpPrefix + "-coordinator-metrics";
@@ -1449,6 +1460,30 @@ private ConsumerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
             metrics.addMetric(metrics.metricName("assigned-partitions",
                 this.metricGrpName,
                 "The number of partitions currently assigned to this consumer"), numParts);
+
+            //HOTFIX - extra liveliness-related metrics
+
+            this.pollInterval = metrics.sensor("poll-interval");
+            this.pollInterval.add(metrics.metricName("poll-interval-avg",
+                this.metricGrpName,
+                "The average time between subsequent poll calls"), new Avg());
+            this.pollInterval.add(metrics.metricName("poll-interval-max",
+                this.metricGrpName,
+                "The max time between subsequent poll calls"), new Max());
+            this.pollInterval.add(createMeter(metrics, metricGrpName, "poll", "poll calls"));
+
+            Measurable lastHeartbeat =
+                new Measurable() {
+                    public double measure(MetricConfig config, long now) {
+                        return TimeUnit.SECONDS.convert(now - prevPollTime, TimeUnit.MILLISECONDS);
+                    }
+                };
+            metrics.addMetric(metrics.metricName("last-poll-seconds-ago",
+                this.metricGrpName,
+                "The number of seconds since the last poll call"),
+                lastHeartbeat);
+
+            //end HOTFIX
         }
     }