KIP-405: 2023-08-21

.github/workflows/pr.yml

@@ -0,0 +1,25 @@
+# The workflow to check pull requests into main.
+# This checks the source in the state as if after the merge.
+name: Pull request checks
+on:
+  pull_request:
+    branches: [ '**' ]
+jobs:
+  build:
+    strategy:
+      matrix:
+        java-version: [ 11, 17 ]
+        runs-on: [ ubuntu-latest ]
+    name: Build on ${{ matrix.runs-on }} with jdk ${{ matrix.java-version }}
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: temurin
+
+      - name: Build Docker image
+        run: make BRANCH=${GITHUB_HEAD_REF} docker_image

Dockerfile

@@ -0,0 +1,38 @@
+##
+# Copyright 2023 Aiven Oy
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# Kafka 3.4.x
+FROM confluentinc/cp-kafka:7.4.0
+
+ARG _SCALA_VERSION
+ARG _KAFKA_VERSION
+ENV _KAFKA_FULL_VERSION "kafka_${_SCALA_VERSION}-${_KAFKA_VERSION}"
+
+USER root
+COPY core/build/distributions/${_KAFKA_FULL_VERSION}.tgz /
+RUN cd / \
+  && tar -xf ${_KAFKA_FULL_VERSION}.tgz \
+  && rm -r /usr/share/java/kafka/* \
+  && cp /${_KAFKA_FULL_VERSION}/libs/* /usr/share/java/kafka/ \
+  && ln -s /usr/share/java/kafka/${_KAFKA_FULL_VERSION}.jar /usr/share/java/kafka/kafka.jar \
+  && rm -r /${_KAFKA_FULL_VERSION}.tgz /${_KAFKA_FULL_VERSION}
+
+# Add test jars with local implementations.
+COPY clients/build/libs/kafka-clients-${_KAFKA_VERSION}-test.jar /usr/share/java/kafka/
+COPY storage/build/libs/kafka-storage-${_KAFKA_VERSION}-test.jar /usr/share/java/kafka/
+COPY storage/api/build/libs/kafka-storage-api-${_KAFKA_VERSION}-test.jar /usr/share/java/kafka/
+
+# Restore the user.
+USER appuser

Makefile

@@ -0,0 +1,39 @@
+##
+# Copyright 2023 Aiven Oy
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+SCALA_VERSION=2.13
+KAFKA_VERSION=3.6.0-SNAPSHOT
+IMAGE_NAME=aivenoy/kafka
+
+BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+IMAGE_TAG := $(subst /,_,$(BRANCH))
+
+.PHONY: clean
+clean:
+	./gradlew clean
+
+core/build/distributions/kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz:
+	./gradlew -PscalaVersion=$(SCALA_VERSION) testJar releaseTarGz
+
+.PHONY: docker_image
+docker_image: core/build/distributions/kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz
+	docker build . \
+		--build-arg _SCALA_VERSION=$(SCALA_VERSION) \
+		--build-arg _KAFKA_VERSION=$(KAFKA_VERSION) \
+		-t $(IMAGE_NAME):$(IMAGE_TAG)
+
+.PHONY: docker_push
+docker_push:
+	docker push $(IMAGE_NAME):$(IMAGE_TAG)

checkstyle/suppressions.xml

@@ -336,9 +336,9 @@
 
     <!-- storage -->
     <suppress checks="CyclomaticComplexity"
-              files="(LogValidator|RemoteLogManagerConfig).java"/>
+              files="(LogValidator|RemoteLogManagerConfig|RemoteLogManager).java"/>
     <suppress checks="NPathComplexity"
-              files="(LogValidator|RemoteIndexCache).java"/>
+              files="(LogValidator|RemoteLogManager|RemoteIndexCache).java"/>
     <suppress checks="ParameterNumber"
               files="(LogAppendInfo|RemoteLogManagerConfig).java"/>
 

core/src/main/scala/kafka/log/LogManager.scala

@@ -430,7 +430,8 @@ class LogManager(logDirs: Seq[File],
               val remainingLogs = decNumRemainingLogs(numRemainingLogs, dir.getAbsolutePath)
               val currentNumLoaded = logsToLoad.length - remainingLogs
               log match {
-                case Some(loadedLog) => info(s"Completed load of $loadedLog with ${loadedLog.numberOfSegments} segments in ${logLoadDurationMs}ms " +
+                case Some(loadedLog) => info(s"Completed load of $loadedLog with ${loadedLog.numberOfSegments} segments, " +
+                  s"local-log-start-offset ${loadedLog.localLogStartOffset()} and log-end-offset ${loadedLog.logEndOffset} in ${logLoadDurationMs}ms " +
                   s"($currentNumLoaded/${logsToLoad.length} completed in $logDirAbsolutePath)")
                 case None => info(s"Error while loading logs in $logDir in ${logLoadDurationMs}ms ($currentNumLoaded/${logsToLoad.length} completed in $logDirAbsolutePath)")
               }