diff --git a/.github/workflows/docker_official_image_build_and_test.yml b/.github/workflows/docker_official_image_build_and_test.yml new file mode 100644 index 0000000000000..a315cd0e0d2ac --- /dev/null +++ b/.github/workflows/docker_official_image_build_and_test.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Docker Official Image Build Test + +on: + workflow_dispatch: + inputs: + image_type: + type: choice + description: Docker image type to build and test + options: + - "jvm" + kafka_version: + description: Kafka version for the docker official image. This should be >=3.7.0 + required: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docker/requirements.txt + - name: Build image and run tests + working-directory: ./docker + run: | + python docker_official_image_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }} + - name: Run CVE scan + uses: aquasecurity/trivy-action@master + with: + image-ref: 'kafka/test:test' + format: 'table' + severity: 'CRITICAL,HIGH' + output: scan_report_${{ github.event.inputs.image_type }}.txt + exit-code: '1' + - name: Upload test report + if: always() + uses: actions/upload-artifact@v3 + with: + name: report_${{ github.event.inputs.image_type }}.html + path: docker/test/report_${{ github.event.inputs.image_type }}.html + - name: Upload CVE scan report + if: always() + uses: actions/upload-artifact@v3 + with: + name: scan_report_${{ github.event.inputs.image_type }}.txt + path: scan_report_${{ github.event.inputs.image_type }}.txt diff --git a/.github/workflows/prepare_docker_official_image_source.yml b/.github/workflows/prepare_docker_official_image_source.yml new file mode 100644 index 0000000000000..45491045836f6 --- /dev/null +++ b/.github/workflows/prepare_docker_official_image_source.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Docker Prepare Docker Official Image Source + +on: + workflow_dispatch: + inputs: + image_type: + type: choice + description: Docker image type to build and test + options: + - "jvm" + kafka_version: + description: Kafka version for the docker official image. This should be >=3.7.0 + required: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docker/requirements.txt + - name: Build Docker Official Image Artifact + working-directory: ./docker + run: | + python prepare_docker_official_image_source.py -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }} + - name: Upload Docker Official Image Artifact + if: success() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.event.inputs.kafka_version }} + path: docker/docker_official_images/${{ github.event.inputs.kafka_version }} diff --git a/build.gradle b/build.gradle index 031389f62c6e9..60ddb955fb63f 100644 --- a/build.gradle +++ b/build.gradle @@ -233,7 +233,8 @@ if (repo != null) { '**/generated/**', 'clients/src/test/resources/serializedData/*', 'docker/test/fixtures/secrets/*', - 'docker/examples/fixtures/secrets/*' + 'docker/examples/fixtures/secrets/*', + 'docker/docker_official_images/.gitkeep' ]) } } else { diff --git a/docker/docker_official_image_build_test.py b/docker/docker_official_image_build_test.py new file mode 100644 index 0000000000000..6ffe25ee0b89c --- /dev/null +++ b/docker/docker_official_image_build_test.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Python script to build and test a docker image +This script is used to generate a test report + +Usage: + docker_official_image_build_test.py --help + Get detailed description of each option + + Example command:- + docker_official_image_build_test.py --image-tag --image-type --kafka-version + + This command will build an image with as image name, as image_tag (it will be latest by default), + as image type (jvm by default), for the kafka version for which the image is being built, and + run tests on the image. + -b can be passed as additional argument if you just want to build the image. + -t can be passed if you just want to run tests on the image. +""" + +import argparse +from distutils.dir_util import copy_tree +import shutil +from common import execute +from docker_build_test import run_docker_tests +import tempfile +import os + + +def build_docker_official_image(image, tag, kafka_version, image_type): + image = f'{image}:{tag}' + current_dir = os.path.dirname(os.path.realpath(__file__)) + temp_dir_path = tempfile.mkdtemp() + copy_tree(f"{current_dir}/docker_official_images/{kafka_version}/{image_type}", + f"{temp_dir_path}/{image_type}") + copy_tree(f"{current_dir}/docker_official_images/{kafka_version}/jvm/resources", + f"{temp_dir_path}/{image_type}/resources") + command = f"docker build -f $DOCKER_FILE -t {image} $DOCKER_DIR" + command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/{image_type}/Dockerfile") + command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/{image_type}") + try: + execute(command.split()) + except: + raise SystemError("Docker Image Build failed") + finally: + shutil.rmtree(temp_dir_path) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "image", help="Image name that you want to keep for the Docker image") + parser.add_argument("--image-tag", "-tag", default="latest", + dest="tag", help="Image tag that you want to add to the image") + parser.add_argument("--image-type", "-type", choices=[ + "jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("--kafka-version", "-v", dest="kafka_version", + help="Kafka version for which the source for docker official image is to be built") + parser.add_argument("--build", "-b", action="store_true", dest="build_only", + default=False, help="Only build the image, don't run tests") + parser.add_argument("--test", "-t", action="store_true", dest="test_only", + default=False, help="Only run the tests, don't build the image") + args = parser.parse_args() + kafka_url = f"https://downloads.apache.org/kafka/{args.kafka_version}/kafka_2.13-{args.kafka_version}.tgz" + if args.build_only or not (args.build_only or args.test_only): + if args.kafka_version: + build_docker_official_image(args.image, args.tag, args.kafka_version, args.image_type) + else: + raise ValueError( + "--kafka-version is required argument for jvm docker official image image") + if args.test_only or not (args.build_only or args.test_only): + run_docker_tests(args.image, args.tag, kafka_url, args.image_type) diff --git a/docker/docker_official_images/.gitkeep b/docker/docker_official_images/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/extract_docker_official_image_artifact.py b/docker/extract_docker_official_image_artifact.py new file mode 100644 index 0000000000000..2d362eb50db94 --- /dev/null +++ b/docker/extract_docker_official_image_artifact.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Python script to extract docker official images artifact and give it executable permissions +This script is used to extract docker official images artifact and give it executable permissions + +Usage: + extract_docker_official_image_artifact.py --help + Get detailed description of each option + + Example command:- + extract_docker_official_image_artifact.py --path_to_downloaded_artifact + + This command will build an extract the downloaded artifact, and copy the contents to the + docker_official_images directory. If the extracted artifact contents already exist in the + docker_official_images directory , they will be overwritten, else they will be created. + +""" +import os +import argparse +import zipfile +import shutil +from pathlib import Path + +def set_executable_permissions(directory): + for root, _, files in os.walk(directory): + for file in files: + path = os.path.join(root, file) + os.chmod(path, os.stat(path).st_mode | 0o111) + + +def extract_artifact(artifact_path): + docker_official_images_dir = Path(os.path.dirname(os.path.realpath(__file__)), "docker_official_images") + temp_dir = Path('temp_extracted') + try: + if temp_dir.exists(): + shutil.rmtree(temp_dir) + temp_dir.mkdir() + with zipfile.ZipFile(artifact_path, 'r') as zip_ref: + zip_ref.extractall(temp_dir) + artifact_version_dirs = list(temp_dir.iterdir()) + if len(artifact_version_dirs) != 1: + raise Exception("Unexpected contents in the artifact. Exactly one version directory is expected.") + artifact_version_dir = artifact_version_dirs[0] + target_version_dir = Path(os.path.join(docker_official_images_dir, artifact_version_dir.name)) + target_version_dir.mkdir(parents=True, exist_ok=True) + for image_type_dir in artifact_version_dir.iterdir(): + target_image_type_dir = Path(os.path.join(target_version_dir, image_type_dir.name)) + if target_image_type_dir.exists(): + shutil.rmtree(target_image_type_dir) + shutil.copytree(image_type_dir, target_image_type_dir) + set_executable_permissions(target_image_type_dir) + finally: + if temp_dir.exists(): + shutil.rmtree(temp_dir) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--path_to_downloaded_artifact", "-artifact_path", required=True, + dest="artifact_path", help="Path to zipped artifacy downloaded from github actions workflow.") + args = parser.parse_args() + extract_artifact(args.artifact_path) diff --git a/docker/generate_kafka_pr_template.py b/docker/generate_kafka_pr_template.py new file mode 100644 index 0000000000000..b3f9577cf89b4 --- /dev/null +++ b/docker/generate_kafka_pr_template.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Python script to prepare the PR template for the docker official image +This script is used to prepare the PR template for the docker official image + +Usage: + Example command:- + generate_kafka_pr_template.py --help + Get detailed description of each option + + generate_kafka_pr_template.py --image-type + + This command will build a PR template for as image type (jvm by default) based docker official image, + on the directories present under docker/docker_official_images. + This PR template will be used to raise a PR in the Docker Official Images Repo. +""" + +import os +import subprocess +import sys +import argparse +from pathlib import Path + + +# Returns the hash of the most recent commit that modified any of the specified files. +def file_commit(*files): + return subprocess.check_output(["git", "log", "-1", "--format=format:%H", "HEAD", "--"] + list(files)).strip().decode('utf-8') + + +# Returns the latest commit hash for all files in a given directory. +def dir_commit(directory): + docker_required_scripts = [str(path) for path in Path(directory).rglob('*') if path.is_file()] + files_to_check = [os.path.join(directory, "Dockerfile")] + docker_required_scripts + return file_commit(*files_to_check) + + +# Split the version string into parts and convert them to integers for version comparision +def get_version_parts(version): + return tuple(int(part) for part in version.name.split('.')) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--image-type", "-type", choices=[ + "jvm"], default="jvm", dest="image_type", help="Image type you want to build") + args = parser.parse_args() + self = os.path.basename(__file__) + current_dir = os.path.dirname(os.path.abspath(__file__)) + docker_official_images_dir = Path(os.path.join(current_dir, "docker_official_images")) + highest_version = "" + + header = f""" +# This file is generated via https://github.com/apache/kafka/blob/{file_commit(os.path.join(current_dir, self))}/docker/generate_kafka_pr_template.py +Maintainers: The Apache Kafka Project (@ApacheKafka) +GitRepo: https://github.com/apache/kafka.git +""" + print(header) + versions = sorted((d for d in docker_official_images_dir.iterdir() if d.is_dir()), key=get_version_parts, reverse=True) + highest_version = max(versions).name if versions else "" + + for dir in versions: + version = dir.name + tags = version + (", latest" if version == highest_version else "") + commit = dir_commit(dir.joinpath(args.image_type)) + + info = f""" +Tags: {tags} +Architectures: amd64,arm64v8 +GitCommit: {commit} +Directory: ./docker/docker_official_images/{version}/{args.image_type} +""" + print(info.strip(), '\n') + + +if __name__ == "__main__": + main() diff --git a/docker/prepare_docker_official_image_source.py b/docker/prepare_docker_official_image_source.py new file mode 100644 index 0000000000000..a39915c9e51f2 --- /dev/null +++ b/docker/prepare_docker_official_image_source.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Python script to prepare the hardcoded source folder for the docker official image +This script is used to prepare the source folder for the docker official image + +Usage: + prepare_docker_official_image_source.py --help + Get detailed description of each option + + Example command:- + prepare_docker_official_image_source.py --image-type --kafka-version + + This command will build a directory with the name as housing the hardcoded static Dockerfile and scripts for + the docker official image, as image type (jvm by default), for the kafka version for which the + image is being built. +""" + +from datetime import date +import argparse +from distutils.dir_util import copy_tree +import os +import shutil + + +def remove_args_and_hardcode_values(file_path, kafka_url): + with open(file_path, 'r') as file: + filedata = file.read() + filedata = filedata.replace("ARG kafka_url", f"ENV kafka_url {kafka_url}") + filedata = filedata.replace( + "ARG build_date", f"ENV build_date {str(date.today())}") + with open(file_path, 'w') as file: + file.write(filedata) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--image-type", "-type", choices=[ + "jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("--kafka-version", "-v", dest="kafka_version", + help="Kafka version for which the source for docker official image is to be built") + args = parser.parse_args() + kafka_url = f"https://downloads.apache.org/kafka/{args.kafka_version}/kafka_2.13-{args.kafka_version}.tgz" + current_dir = os.path.dirname(os.path.realpath(__file__)) + new_dir = os.path.join( + current_dir, f'docker_official_images', args.kafka_version) + if os.path.exists(new_dir): + shutil.rmtree(new_dir) + os.makedirs(new_dir) + copy_tree(os.path.join(current_dir, args.image_type), os.path.join(new_dir, args.kafka_version, args.image_type)) + copy_tree(os.path.join(current_dir, 'resources'), os.path.join(new_dir, args.kafka_version, args.image_type, 'resources')) + remove_args_and_hardcode_values( + os.path.join(new_dir, args.kafka_version, args.image_type, 'Dockerfile'), kafka_url)