[INTERNAL] [BUILD] Publish docker images to adobe/kafka-operator and …

…adobe/kafka docker hub repos

- build the koperator docker image
- build Apache kafka docker image
When `kafka-*` tags are created a github action is triggered
to build and push a new adobe/kafka docker image version

.github/workflows/build-push-docker.yml

@@ -0,0 +1,26 @@
+name: build-publish-docker-image
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get tag name
+        id: vars
+        run: echo ::set-output name=tag::${GITHUB_REF:10}
+      - uses: docker/build-push-action@v1
+        if: ${{ !startsWith( steps.vars.outputs.tag, 'kafka-' ) }}
+        with:
+          dockerfile: Dockerfile
+          build_args: VERSION=${{ steps.vars.outputs.tag }},GIT_SHA=${{ github.sha }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: adobe/kafka-operator
+          tag_with_ref: true
+          push: ${{ startsWith(github.ref, 'refs/tags/') }}
+          add_git_labels: true

.github/workflows/build-push-kafka-docker.yml

@@ -0,0 +1,44 @@
+name: build-publish-kafka-docker-image
+
+on:
+  push:
+    tags:
+      - 'kafka-*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=adobe/kafka
+          VERSION=latest
+          if [[ $GITHUB_REF == refs/tags/kafka-* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/kafka-}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION}"
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: docker/kafka
+          push: ${{ startsWith(github.ref, 'refs/tags/') }}
+          tags: ${{ steps.prep.outputs.tags }}
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '34 23 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/helm.yml

@@ -65,13 +65,7 @@ jobs:
       - name: Set Helm push enabled
         id: set-helm-push-enabled
         run: |
-          HELM_PUSH_ENABLED=""
-          if [ "${{ github.event_name }}" == "push" ] && echo "${{ steps.set-git-refname.outputs.git_refname }}" | grep -E -q "^chart/${{ env.HELM_CHART_NAME }}/[0-9]+.[0-9]+.[0-9]+**"; then
-            HELM_PUSH_ENABLED=1
-          else
-            printf >&2 "Unstable chart (%s) from %s event, chart will not be pushed" "${{ steps.set-git-refname.outputs.git_refname }}" "${{ github.event_name }}"
-          fi
-
+          HELM_PUSH_ENABLED="0"
           echo "HELM_PUSH_ENABLED=${HELM_PUSH_ENABLED}"
           echo "helm_push_enabled=${HELM_PUSH_ENABLED}" >> $GITHUB_OUTPUT
 

docker/kafka/Dockerfile

@@ -0,0 +1,44 @@
+FROM alpine:latest AS kafka_dist
+
+ARG scala_version=2.13
+ARG kafka_version=2.6.2
+ARG kafka_distro_base_url=https://downloads.apache.org/kafka
+
+ENV kafka_distro=kafka_$scala_version-$kafka_version.tgz
+ENV kafka_distro_asc=$kafka_distro.asc
+
+RUN apk add --no-cache gnupg
+
+WORKDIR /var/tmp
+
+RUN wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro
+RUN wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro_asc
+RUN wget -q $kafka_distro_base_url/KEYS
+
+RUN gpg --import KEYS
+RUN gpg --verify $kafka_distro_asc $kafka_distro
+
+RUN tar -xzf $kafka_distro
+RUN rm -r kafka_$scala_version-$kafka_version/bin/windows
+
+
+FROM azul/zulu-openjdk:16.0.1
+
+ARG scala_version=2.13
+ARG kafka_version=2.6.2
+
+ENV KAFKA_VERSION=$kafka_version \
+    SCALA_VERSION=$scala_version \
+    KAFKA_HOME=/opt/kafka
+
+ENV PATH=${PATH}:${KAFKA_HOME}/bin
+
+RUN mkdir ${KAFKA_HOME} && apt-get update && apt-get -y upgrade && apt-get install curl -y && apt-get clean
+
+COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$kafka_version ${KAFKA_HOME}
+COPY opt/kafka/config/log4j.properties ${KAFKA_HOME}/config/log4j.properties
+
+
+RUN chmod a+x ${KAFKA_HOME}/bin/*.sh
+
+CMD ["kafka-server-start.sh"]

docker/kafka/README.md

@@ -0,0 +1,13 @@
+# Kafka docker image
+
+`adobe/kafka` docker image build configuration.
+
+A new `kafka-*` tag created in this repo triggers the image build and push to [adobe/kafka](https://hub.docker.com/r/adobe/kafka/tags?page=1&ordering=last_updated) docker hub repo.
+
+Tags should be `kafka-<scala_version>-<kafka_version>` e.g `kafak-2.13-2.6.2`
+
+# Upstream base
+
+This is based on [wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) with the following additions:
+1. Use `openjdk 16` in order to support container based resource monitoring
+2. Use custom `log4j.properties` to get all kafka logs to stdout only

docker/kafka/opt/kafka/config/log4j.properties

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Unspecified loggers and loggers with additivity=true output to server.log and stdout
+# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
+log4j.rootLogger=INFO, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Change the line below to adjust ZK client logging
+log4j.logger.org.apache.zookeeper=INFO
+
+# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
+log4j.logger.kafka=INFO
+log4j.logger.org.apache.kafka=INFO
+
+# Change to DEBUG or TRACE to enable request logging
+log4j.logger.kafka.request.logger=WARN
+
+# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
+# related to the handling of requests
+#log4j.logger.kafka.network.Processor=TRACE, requestAppender
+#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
+#
+log4j.logger.kafka.network.RequestChannel$=WARN
+
+log4j.logger.kafka.controller=DEBUG
+
+
+log4j.logger.kafka.log.LogCleaner=INFO
+
+
+log4j.logger.state.change.logger=INFO
+
+# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses
+log4j.logger.kafka.authorizer.logger=INFO