Master

.gitignore

@@ -21,3 +21,4 @@ java/.classpath
 java/.project
 java/.settings/
 .vscode/*
+examples/cp-all-in-one-cloud/delta_configs/

CONTRIBUTING.md

@@ -0,0 +1,125 @@
+# Contributing
+
+## General Guidelines
+
+When submitting a pull request (PR), please use the following
+guidelines:
+
+-   Make sure your code respects existing formatting conventions. In
+    general, follow the same coding style as the code that you are
+    modifying.
+-   Do add/update documentation appropriately for the change you are
+    making.
+-   If you are introducing a new feature you may want to first submit
+    your idea for feedback to the [Confluent mailing
+    list](mailto:[email protected]).
+-   Non-trivial features should include unit tests covering the new
+    functionality.
+-   Bugfixes should include a unit test or integration test reproducing
+    the issue.
+-   Try to keep pull requests short and submit separate ones for
+    unrelated features, but feel free to combine simple bugfixes/tests
+    into one pull request.
+-   Keep the number of commits small and combine commits for related
+    changes.
+-   Each commit should compile on its own and ideally pass tests.
+-   Keep formatting changes in separate commits to make code reviews
+    easier and distinguish them from actual code changes.
+
+## GitHub Workflow
+
+1.  Fork the confluentinc/cp-docker-images repository into your GitHub
+    account
+
+    > <https://github.com/confluentinc/cp-docker-images/fork>
+
+2.  Clone your fork of the GitHub repository
+
+    ```
+    $ git clone <[email protected]>:\<username\>/cp-docker-images.git
+    ```
+
+    > Note: replace \<username\> with your GitHub username.
+
+3.  Add a remote to keep up with upstream changes
+
+    ```
+    $ git remote add upstream https://github.com/confluentinc/cp-docker-images.git
+    ```
+
+    > If you already have a copy, fetch upstream changes
+
+    ```
+    git fetch upstream
+    ```
+
+4.  Create a feature branch to work in
+
+    ```
+    $ git checkout -b feature-xxx remotes/upstream/master
+    ```
+
+5.  Work in your feature branch
+
+    ```
+    $ git commit -a
+    ```
+
+6.  Periodically rebase your changes
+
+    ```
+    $ git pull \--rebase
+    ```
+
+7.  When done, combine (\"squash\") related commits into a single one
+
+    ```
+    $ git rebase -i upstream/master
+    ```
+    >
+    > This will open your editor and allow you to re-order commits and merge
+    > them: - Re-order the lines to change commit order (to the extent
+    > possible without creating conflicts) - Prefix commits using s (squash)
+    > or f (fixup) to merge extraneous commits.
+
+8.  Submit a pull-request
+
+    ```
+    $ git push origin feature-xxx
+    ```
+    >
+    > Go to your cp-docker-images fork main page
+
+    ``` 
+    <https://github.com/>\<username\>/cp-docker-images
+    ```
+    >
+    > If you recently pushed your changes GitHub will automatically pop up a
+    > Compare & pull request button for any branches you recently pushed to.
+    > If you click that button it will automatically offer you to submit
+    > your pull-request to the confluentinc/cp-docker-images repository.
+    >
+    > -   Give your pull-request a meaningful title.
+    > -   In the description, explain your changes and the problem they are
+    >     solving.
+
+9.  Addressing code review comments
+
+    > Repeat steps 5. through 7. to address any code review comments and
+    > rebase your changes if necessary.
+    >
+    > Push your updated changes to update the pull request
+    >
+    ```
+    $ git push origin \[\--force\] feature-xxx
+    ```
+    >
+    > --force may be necessary to overwrite your existing pull request in
+    > case your commit history was changed when performing the rebase.
+    >
+    > Note: Be careful when using \--force since you may lose data if you
+    > are not careful.
+    >
+    ```
+    $ git push origin \--force feature-xxx
+    ```

java/README.md → DOCKER_UTILS.md

@@ -26,16 +26,9 @@ This command sends a metadata query to the broker and verifies that expected num
 
 		 mvn clean compile assembly:single
 
-2. See `src/test/bin/cli-test.sh` for examples on running the commands.
-
 ## How to test ?
 
 1. JUnit tests: mvn test
-2. Shell test :
-
-        cd docker-utils
-        src/test/bin/cli-test.sh
-
 
 ## Client.properties
 

Makefile

@@ -2,22 +2,22 @@
 -include local.make
 
 # Bump this on subsequent build, reset on new version or public release. Inherit from env for CI builds.
-BUILD_NUMBER ?= 3
+BUILD_NUMBER ?= 1
 
-CONFLUENT_MAJOR_VERSION ?= 4
-CONFLUENT_MINOR_VERSION ?= 1
+CONFLUENT_MAJOR_VERSION ?= 5
+CONFLUENT_MINOR_VERSION ?= 4
 CONFLUENT_PATCH_VERSION ?= 0
 
 CONFLUENT_VERSION ?= ${CONFLUENT_MAJOR_VERSION}.${CONFLUENT_MINOR_VERSION}.${CONFLUENT_PATCH_VERSION}
 
-KAFKA_VERSION ?= 1.1.0
+KAFKA_VERSION ?= 2.4.0
 
-COMPONENTS := base zookeeper kafka kafka-rest schema-registry kafka-connect-base kafka-connect enterprise-control-center kafkacat enterprise-replicator enterprise-kafka
+COMPONENTS := base zookeeper kafka server kafka-rest schema-registry kafka-connect-base kafka-connect server-connect-base server-connect enterprise-control-center kafkacat enterprise-replicator enterprise-replicator-executable enterprise-kafka kafka-mqtt
 COMMIT_ID := $(shell git rev-parse --short HEAD)
 MYSQL_DRIVER_VERSION := 5.1.39
 
 # Set this variable externally to point at a different repo, such as when building SNAPSHOT images
-CONFLUENT_PACKAGES_REPO ?= http://packages.confluent.io
+CONFLUENT_PACKAGES_REPO ?= https://packages.confluent.io
 
 # Set to false for public releases
 ALLOW_UNSIGNED ?= false
@@ -48,16 +48,14 @@ clean-images:
 
 debian/base/include/etc/confluent/docker/docker-utils.jar:
 	mkdir -p debian/base/include/etc/confluent/docker
-	cd java \
-	&& mvn clean compile package assembly:single -DskipTests \
-	&& cp target/docker-utils-${CONFLUENT_VERSION}${CONFLUENT_MVN_LABEL}-jar-with-dependencies.jar ../debian/base/include/etc/confluent/docker/docker-utils.jar \
-	&& cd -
+	mvn -U clean compile package -DskipTests \
+	&& cp target/docker-utils-${CONFLUENT_VERSION}${CONFLUENT_MVN_LABEL}-jar-with-dependencies.jar debian/base/include/etc/confluent/docker/docker-utils.jar
 
 build-debian: debian/base/include/etc/confluent/docker/docker-utils.jar
 	COMPONENTS="${COMPONENTS}" \
 	ALLOW_UNSIGNED=${ALLOW_UNSIGNED} \
 	CONFLUENT_PACKAGES_REPO=${CONFLUENT_PACKAGES_REPO} \
-    KAFKA_VERSION=${KAFKA_VERSION} \
+	KAFKA_VERSION=${KAFKA_VERSION} \
 	CONFLUENT_MVN_LABEL=${CONFLUENT_MVN_LABEL} \
 	CONFLUENT_DEB_LABEL=${CONFLUENT_DEB_LABEL} \
 	CONFLUENT_RPM_LABEL=${CONFLUENT_RPM_LABEL} \
@@ -116,14 +114,6 @@ venv/bin/activate: tests/requirements.txt
 	venv/bin/pip install -Ur tests/requirements.txt
 	touch venv/bin/activate
 
-test-docker-utils:
-	mkdir -p ../debian/base/include/etc/confluent/docker
-	cd java \
-	&& mvn clean compile package assembly:single \
-	&& src/test/bin/cli-test.sh \
-	&& cp target/docker-utils-${CONFLUENT_VERSION}${CONFLUENT_MVN_LABEL}-jar-with-dependencies.jar ../debian/base/include/etc/confluent/docker/docker-utils.jar \
-	&& cd -
-
 test-build: venv clean build-debian build-test-images
 	IMAGE_DIR=$(pwd) venv/bin/py.test tests/test_build.py -v
 
@@ -158,7 +148,6 @@ test-control-center: venv clean-containers build-debian build-test-images
 test-all: \
 	venv \
 	clean \
-	test-docker-utils \
 	build-debian \
 	build-test-images \
 	test-build \

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+### Deprecation Notice
+
+This is used for building images for version 4.1.x or lower, and should not be used for adding new images.
+
+For the 5.4.0 release and greater the images have been migrated to the following repositories:
+
+* [common-docker](https://github.com/confluentinc/common-docker)
+* [control-center-images](https://github.com/confluentinc/control-center-images)
+* [kafkacat-images](https://github.com/confluentinc/kafkacat-images)
+* [kafka-images](https://github.com/confluentinc/kafka-images)
+* [kafka-mqtt-images](https://github.com/confluentinc/kafka-mqtt-images)
+* [kafka-replicator-images](https://github.com/confluentinc/kafka-replicator-images)
+* [kafka-rest-images](https://github.com/confluentinc/kafka-rest-images)
+* [kafka-streams-examples](https://github.com/confluentinc/kafka-streams-examples)
+* [ksql-images](https://github.com/confluentinc/ksql-images)
+* [schema-registry-images](https://github.com/confluentinc/schema-registry-images)

README.md

@@ -1,20 +1,51 @@
-Docker Images for Confluent Plaform
+Deprecation Notice
 ===
 
+This is used for building images for version 5.3.x or lower, and should not be used for adding new images.
+
+For the 5.4.0 release and greater the images have been migrated to the following repositories:
+
+* [common-docker](https://github.com/confluentinc/common-docker)
+* [control-center-images](https://github.com/confluentinc/control-center-images)
+* [kafkacat-images](https://github.com/confluentinc/kafkacat-images)
+* [kafka-images](https://github.com/confluentinc/kafka-images)
+* [kafka-mqtt-images](https://github.com/confluentinc/kafka-mqtt-images)
+* [kafka-replicator-images](https://github.com/confluentinc/kafka-replicator-images)
+* [kafka-rest-images](https://github.com/confluentinc/kafka-rest-images)
+* [kafka-streams-examples](https://github.com/confluentinc/kafka-streams-examples)
+* [ksql-images](https://github.com/confluentinc/ksql-images)
+* [schema-registry-images](https://github.com/confluentinc/schema-registry-images)
+
+# Docker Images for Confluent Plaform
+
 Docker images for deploying and running the Confluent Platform.  The images are currently available on [DockerHub](https://hub.docker.com/u/confluentinc/).  They are currently only available for Confluent Platform 3.0.1 and after.
 
-Full documentation for using the images can be found [here](http://docs.confluent.io/current/cp-docker-images/docs/intro.html).
+Full documentation for using the images can be found [here](https://docs.confluent.io/current/installation/docker/index.html).
+
+# Networking and Kafka on Docker
+
+When running Kafka under Docker, you need to pay careful attention to your configuration of hosts and ports to enable components both internal and external to the docker network to communicate. You can see more details in [this article](https://rmoff.net/2018/08/02/kafka-listeners-explained/).
 
-# Important Caveat - Images Not Tested for Docker for Mac or Windows
+# Known issues on Mac/Windows
 
-These images are not tested on Docker for Mac or Docker for Windows. These images will be updated in the near future to support these platforms. For more details on these known issues, you can refer to the following links:
+For more details on known issues when running these images on Mac/Windows, you can refer to the following links:
 
 * [Hostname Issue](https://forums.docker.com/t/docker-for-mac-does-not-add-docker-hostname-to-etc-hosts/8620/4)
 * Host networking on Docker for Mac: [link 1](https://forums.docker.com/t/should-docker-run-net-host-work/14215), [link 2](https://forums.docker.com/t/net-host-does-not-work/17378/7), [link 3](https://forums.docker.com/t/explain-networking-known-limitations-explain-host/15205/4)
 
+# Building
+
+Use `make` to perform various builds and tests
+
+
+# Docker Utils
+
+See [Docker Utils](DOCKER_UTILS.md)
+
+
 # Contribute
 
-Start by reading our guidelines on contributing to this project found [here](http://docs.confluent.io/current/cp-docker-images/docs/contributing.html).
+Start by reading our guidelines on contributing to this project found [here](CONTRIBUTING.md).
 
 - Source Code: https://github.com/confluentinc/cp-docker-images
 - Issue Tracker: https://github.com/confluentinc/cp-docker-images/issues

bin/build-debian

@@ -1,14 +1,29 @@
 #!/bin/bash
+set -x
+
+# todo: added for debugging. to be removed once integration is complete
+echo "------------- ENV variables --------------------"
+echo "COMPONENTS=${COMPONENTS}" 
+echo "ALLOW_UNSIGNED=${ALLOW_UNSIGNED} "
+echo "CONFLUENT_PACKAGES_REPO=${CONFLUENT_PACKAGES_REPO}"
+echo "KAFKA_VERSION=${KAFKA_VERSION}"
+echo "CONFLUENT_MVN_LABEL=${CONFLUENT_MVN_LABEL}"
+echo "CONFLUENT_DEB_LABEL=${CONFLUENT_DEB_LABEL}"
+echo "CONFLUENT_RPM_LABEL=${CONFLUENT_RPM_LABEL}"
+echo "CONFLUENT_MAJOR_VERSION=${CONFLUENT_MAJOR_VERSION}"
+echo "CONFLUENT_MINOR_VERSION=${CONFLUENT_MINOR_VERSION}"
+echo "CONFLUENT_PATCH_VERSION=${CONFLUENT_PATCH_VERSION}"
+echo "CONFLUENT_VERSION=${CONFLUENT_VERSION}"
+echo "VERSION=${VERSION}"
+echo "COMMIT_ID=${COMMIT_ID}"
+echo "BUILD_NUMBER=${BUILD_NUMBER}"
+echo "REPOSITORY=${REPOSITORY}"
+echo "RELEASE_QUALITY=${RELEASE_QUALITY}"
 
 for component in ${COMPONENTS} ; do
     echo "\n\nBuilding ${component} \n==========================================\n "
 
-    if [ "${component}" = "base" ]; then
-        BUILD_ARGS="--build-arg ALLOW_UNSIGNED=${ALLOW_UNSIGNED} --build-arg CONFLUENT_PACKAGES_REPO=${CONFLUENT_PACKAGES_REPO} --build-arg CONFLUENT_MVN_LABEL=${CONFLUENT_MVN_LABEL}"
-
-    else
-        BUILD_ARGS=""
-    fi
+    BUILD_ARGS="--build-arg ALLOW_UNSIGNED=${ALLOW_UNSIGNED} --build-arg CONFLUENT_PACKAGES_REPO=${CONFLUENT_PACKAGES_REPO} --build-arg CONFLUENT_MVN_LABEL=${CONFLUENT_MVN_LABEL}"
 
     for type in "" rpm; do
         DOCKER_FILE="debian/${component}/Dockerfile"
@@ -18,7 +33,6 @@ for component in ${COMPONENTS} ; do
             COMPONENT_NAME="rpm-${component}"
             DOCKER_FILE="${DOCKER_FILE}.rpm"
             CONFLUENT_PLATFORM_LABEL=${CONFLUENT_RPM_LABEL}
-
         else
             CONFLUENT_PLATFORM_LABEL=${CONFLUENT_DEB_LABEL}
         fi
@@ -33,4 +47,3 @@ for component in ${COMPONENTS} ; do
         fi;
     done
 done
-