Kafka-15444: Native docker image for Apache Kafka(KIP-974)

core/src/main/scala/kafka/docker/KafkaDockerWrapper.scala

@@ -16,6 +16,7 @@
  */
 package kafka.docker
 
+import kafka.Kafka
 import kafka.tools.StorageTool
 import kafka.utils.Exit
 import net.sourceforge.argparse4j.ArgumentParsers
@@ -45,6 +46,9 @@ object KafkaDockerWrapper {
 
         val formatCmd = formatStorageCmd(finalConfigsPath, envVars)
         StorageTool.main(formatCmd)
+      case "start" =>
+        val configFile = namespace.getString("config")
+        Kafka.main(Array(configFile))
       case _ =>
         throw new RuntimeException(s"Unknown operation $command. " +
           s"Please provide a valid operation: 'setup'.")
@@ -60,7 +64,13 @@ object KafkaDockerWrapper {
 
     val subparsers = parser.addSubparsers().dest("command")
 
-    val setupParser = subparsers.addParser("setup")
+    val kafkaStartParser = subparsers.addParser("start").help("Start kafka server.")
+    kafkaStartParser.addArgument("--config", "-C")
+      .action(store())
+      .required(true)
+      .help("The kafka server configuration file")
+
+    val setupParser = subparsers.addParser("setup").help("Setup property files and format storage.")
 
     setupParser.addArgument("--default-configs-dir", "-D").
       action(store()).

docker/common.py

@@ -31,13 +31,13 @@ def get_input(message):
         raise ValueError("This field cannot be empty")
     return value
 
-def jvm_image(command):
+def build_docker_image_runner(command, image_type):
     temp_dir_path = tempfile.mkdtemp()
     current_dir = os.path.dirname(os.path.realpath(__file__))
-    copy_tree(f"{current_dir}/jvm", f"{temp_dir_path}/jvm")
-    copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/jvm/resources")
-    command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/jvm/Dockerfile")
-    command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/jvm")
+    copy_tree(f"{current_dir}/{image_type}", f"{temp_dir_path}/{image_type}")
+    copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/{image_type}/resources")
+    command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/{image_type}/Dockerfile")
+    command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}")
     try:
         execute(command.split())
     except:

docker/docker_build_test.py

@@ -37,28 +37,28 @@
 from distutils.dir_util import copy_tree
 import shutil
 from test.docker_sanity_test import run_tests
-from common import execute, jvm_image
+from common import execute, build_docker_image_runner
 import tempfile
 import os
 
-def build_jvm(image, tag, kafka_url):
+def build_docker_image(image, tag, kafka_url, image_type):
     image = f'{image}:{tag}'
-    jvm_image(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR")
+    build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR", image_type)
 
-def run_jvm_tests(image, tag, kafka_url):
+def run_docker_tests(image, tag, kafka_url, image_type):
     temp_dir_path = tempfile.mkdtemp()
     try:
         current_dir = os.path.dirname(os.path.realpath(__file__))
         copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures")
         execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url])
         execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"])
         execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"])
-        failure_count = run_tests(f"{image}:{tag}", "jvm", temp_dir_path)
+        failure_count = run_tests(f"{image}:{tag}", image_type, temp_dir_path)
     except:
         raise SystemError("Failed to run the tests")
     finally:
         shutil.rmtree(temp_dir_path)
-    test_report_location_text = f"To view test report please check {current_dir}/test/report_jvm.html"
+    test_report_location_text = f"To view test report please check {current_dir}/test/report_{image_type}.html"
     if failure_count != 0:
         raise SystemError(f"{failure_count} tests have failed. {test_report_location_text}")
     else:
@@ -68,17 +68,17 @@ def run_jvm_tests(image, tag, kafka_url):
     parser = argparse.ArgumentParser()
     parser.add_argument("image", help="Image name that you want to keep for the Docker image")
     parser.add_argument("--image-tag", "-tag", default="latest", dest="tag", help="Image tag that you want to add to the image")
-    parser.add_argument("--image-type", "-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build")
+    parser.add_argument("--image-type", "-type", choices=["jvm", "native"], default="jvm", dest="image_type", help="Image type you want to build")
     parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image")
     parser.add_argument("--build", "-b", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests")
     parser.add_argument("--test", "-t", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image")
     args = parser.parse_args()
 
-    if args.image_type == "jvm" and (args.build_only or not (args.build_only or args.test_only)):
+    if args.build_only or not (args.build_only or args.test_only):
         if args.kafka_url:
-            build_jvm(args.image, args.tag, args.kafka_url)
+            build_docker_image(args.image, args.tag, args.kafka_url, args.image_type)
         else:
-            raise ValueError("--kafka-url is a required argument for jvm image")
+            raise ValueError("--kafka-url is a required argument for docker image")
 
-    if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)):
-        run_jvm_tests(args.image, args.tag, args.kafka_url)
+    if args.test_only or not (args.build_only or args.test_only):
+        run_docker_tests(args.image, args.tag, args.kafka_url, args.image_type)

docker/docker_release.py

@@ -38,13 +38,13 @@
 from datetime import date
 import argparse
 
-from common import execute, jvm_image
+from common import execute, build_docker_image_runner
 
-def build_push_jvm(image, kafka_url):
+def build_push(image, kafka_url, image_type):
     try:
         create_builder()
-        jvm_image(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \
-              --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR")
+        build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \
+              --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR", image_type)
     except:
         raise SystemError("Docker image push failed")
     finally:
@@ -63,13 +63,12 @@ def remove_builder():
           Please ensure you are logged in the docker registry that you are trying to push to.")
     parser = argparse.ArgumentParser()
     parser.add_argument("image", help="Dockerhub image that you want to push to (in the format <registry>/<namespace>/<image_name>:<image_tag>)")
-    parser.add_argument("--image-type", "-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build")
+    parser.add_argument("--image-type", "-type", choices=["jvm", "native"], default="jvm", dest="image_type", help="Image type you want to build")
     parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image")
     args = parser.parse_args()
 
     print(f"Docker image of type {args.image_type} containing kafka downloaded from {args.kafka_url} will be pushed to {args.image}")
 
     print("Building and pushing the image")
-    if args.image_type == "jvm":
-        build_push_jvm(args.image, args.kafka_url)
+    build_push(args.image, args.kafka_url, args.image_type)
     print(f"Image has been pushed to {args.image}")

docker/native/Dockerfile

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ghcr.io/graalvm/graalvm-community:21 AS build-native-image
+
+ARG kafka_url
+
+WORKDIR /app
+
+ENV KAFKA_URL=$kafka_url
+COPY native-image-configs native-image-configs
+
+RUN mkdir kafka; \
+    microdnf install wget; \
+    wget -nv -O kafka.tgz "$KAFKA_URL"; \
+    wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
+    tar xfz kafka.tgz -C kafka --strip-components 1; \
+    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
+    gpg --import KEYS; \
+    gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    rm kafka.tgz ; \
+    cd kafka ;  \
+# Build the native-binary of the apache kafka using graalVM native-image.
+    native-image --no-fallback \
+        --enable-http \
+        --enable-https \
+        --allow-incomplete-classpath \
+        --report-unsupported-elements-at-runtime \
+        --install-exit-handlers \
+        --enable-monitoring=jmxserver,jmxclient,heapdump,jvmstat \
+        -H:+ReportExceptionStackTraces \
+        -H:+EnableAllSecurityServices \
+        -H:EnableURLProtocols=http,https \
+        -H:AdditionalSecurityProviders=sun.security.jgss.SunProvider \
+        -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \
+        -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \
+        -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \
+        -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \
+        -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \
+        -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \
+        --verbose \
+        -march=compatibility \
+        -cp "libs/*" kafka.docker.KafkaDockerWrapper \
+        -o kafka.Kafka
+
+
+FROM alpine:latest
+
+EXPOSE 9092
+
+ARG build_date
+
+LABEL org.label-schema.name="kafka" \
+      org.label-schema.description="Apache Kafka" \
+      org.label-schema.build-date="${build_date}" \
+      org.label-schema.vcs-url="https://github.com/apache/kafka" \
+      maintainer="Apache Kafka"
+
+RUN apk update ; \
+    apk add --no-cache gcompat ; \
+    apk add --no-cache bash ; \
+    mkdir -p /etc/kafka/docker /mnt/shared/config /opt/kafka/config /etc/kafka/secrets ; \
+    adduser -h /home/appuser -D --shell /bin/bash appuser ; \
+    chown appuser:root -R /etc/kafka /opt/kafka /mnt/shared/config ; \
+    chmod -R ug+w /etc/kafka /opt/kafka /mnt/shared/config ;
+
+COPY --chown=appuser:root --from=build-native-image /app/kafka/kafka.Kafka /opt/kafka/
+COPY --chown=appuser:root --from=build-native-image /app/kafka/config/kraft/server.properties /etc/kafka/docker/
+COPY --chown=appuser:root --from=build-native-image /app/kafka/config/log4j.properties /etc/kafka/docker/
+COPY --chown=appuser:root --from=build-native-image /app/kafka/config/tools-log4j.properties /etc/kafka/docker/
+COPY --chown=appuser:root resources/common-scripts /etc/kafka/docker/
+COPY --chown=appuser:root launch /etc/kafka/docker/
+
+USER appuser
+
+VOLUME ["/etc/kafka/secrets", "/mnt/shared/config"]
+
+CMD ["/etc/kafka/docker/run"]

docker/native/launch

@@ -0,0 +1,50 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property.
+if [ -z "${KAFKA_JMX_OPTS-}" ]; then
+    export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true \
+    -Dcom.sun.management.jmxremote.authenticate=false \
+    -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# The JMX client needs to be able to connect to java.rmi.server.hostname.
+# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container.
+# For host n/w, this is the IP that the hostname on the host resolves to.
+
+# If you have more than one n/w configured, hostname -i gives you all the IPs,
+# the default is to pick the first IP (or network).
+export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)}
+
+if [ "${KAFKA_JMX_PORT-}" ]; then
+    export JMX_PORT=$KAFKA_JMX_PORT
+    export KAFKA_JMX_OPTS="${KAFKA_JMX_OPTS-} -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME \
+    -Dcom.sun.management.jmxremote.local.only=false \
+    -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT \
+    -Dcom.sun.management.jmxremote.port=$JMX_PORT"
+fi
+
+# Invoke the docker wrapper to setup property files and format storage
+result=$(/opt/kafka/kafka.Kafka setup \
+      --default-configs-dir /etc/kafka/docker \
+      --mounted-configs-dir /mnt/shared/config \
+      --final-configs-dir /opt/kafka/config \
+      -Dlog4j.configuration=file:/opt/kafka/config/tools-log4j.properties 2>&1) || \
+      echo $result | grep -i "already formatted" || \
+      { echo $result && (exit 1) }
+
+KAFKA_LOG4J_CMD_OPTS="-Dkafka.logs.dir=/opt/kafka/logs/ -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties"
+exec /opt/kafka/kafka.Kafka start --config /opt/kafka/config/server.properties $KAFKA_LOG4J_CMD_OPTS $KAFKA_JMX_OPTS ${KAFKA_OPTS-}

docker/native/native-image-configs/jni-config.json

@@ -0,0 +1,35 @@
+[
+{
+  "name":"[Lcom.sun.management.internal.DiagnosticCommandArgumentInfo;"
+},
+{
+  "name":"[Lcom.sun.management.internal.DiagnosticCommandInfo;"
+},
+{
+  "name":"com.github.luben.zstd.ZstdInputStreamNoFinalizer",
+  "fields":[{"name":"dstPos"}, {"name":"srcPos"}]
+},
+{
+  "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }]
+},
+{
+  "name":"com.sun.management.internal.DiagnosticCommandInfo",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }]
+},
+{
+  "name":"java.lang.Boolean",
+  "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }]
+},
+{
+  "name":"java.lang.OutOfMemoryError"
+},
+{
+  "name":"java.util.Arrays",
+  "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }]
+},
+{
+  "name":"sun.management.VMManagementImpl",
+  "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}]
+}
+]

docker/native/native-image-configs/predefined-classes-config.json

@@ -0,0 +1,7 @@
+[
+  {
+    "type":"agent-extracted",
+    "classes":[
+    ]
+  }
+]

docker/native/native-image-configs/proxy-config.json

@@ -0,0 +1,5 @@
+[
+  {
+    "interfaces":["sun.misc.SignalHandler"]
+  }
+]