[incubator/kafka]: add reassignPartitions to topic configuration (#7623)

* [incubator/kafka]: add reassignPartitions to topic configuration

Signed-off-by: Giacomo Guiulfo <[email protected]>

* [incubator/kafka]: bump chart to 0.10.0

Signed-off-by: Giacomo Guiulfo <[email protected]>

Author: giacomoguiulfo

incubator/kafka/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 description: Apache Kafka is publish-subscribe messaging rethought as a distributed
   commit log.
 name: kafka
-version: 0.9.6
+version: 0.10.0
 appVersion: 4.1.2
 keywords:
 - kafka

incubator/kafka/README.md

@@ -115,7 +115,7 @@ following configurable parameters:
 | `prometheus.operator.enabled`                  | True if using the Prometheus Operator, False if not                                                                                                                      | `false`                                                            |
 | `prometheus.operator.serviceMonitor.namespace` | Namespace which Prometheus is running in.  Default to kube-prometheus install.                                                                                           | `monitoring`                                                       |
 | `prometheus.operator.serviceMonitor.selector`  | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install                                                               | `{ prometheus: kube-prometheus }`                                  |
-| `topics`                                       | List of topics to create & configure. Can specify name, partitions, replicationFactor, config. See values.yaml                                                           | `[]` (Empty list)                                                  |
+| `topics`                                       | List of topics to create & configure. Can specify name, partitions, replicationFactor, reassignPartitions, config. See values.yaml                                       | `[]` (Empty list)                                                  |
 | `zookeeper.enabled`                            | If True, installs Zookeeper Chart                                                                                                                                        | `true`                                                             |
 | `zookeeper.resources`                          | Zookeeper resource requests and limits                                                                                                                                   | `{}`                                                               |
 | `zookeeper.env`                                | Environmental variables provided to Zookeeper Zookeeper                                                                                                                  | `{ZK_HEAP_SIZE: "1G"}`                                             |

incubator/kafka/templates/configmap-config.yaml

@@ -26,7 +26,19 @@ data:
     done
     echo "Applying runtime configuration using {{ .Values.image }}:{{ .Values.imageTag }}"
     {{- range $n, $topic := .Values.topics }}
-    {{- if and $topic.partitions $topic.replicationFactor }}
+    {{- if and $topic.partitions $topic.replicationFactor $topic.reassignPartitions }}
+    cat << EOF > {{ $topic.name }}-increase-replication-factor.json
+      {"version":1, "partitions":[
+        {{- $partitions := (int $topic.partitions) }}
+        {{- $replicas := (int $topic.replicationFactor) }}
+        {{- range $i := until $partitions }}
+          {"topic":"{{ $topic.name }}","partition":{{ $i }},"replicas":[{{- range $j := until $replicas }}{{ $j }}{{- if ne $j (sub $replicas 1) }},{{- end }}{{- end }}]}{{- if ne $i (sub $partitions 1) }},{{- end }}
+        {{- end }}
+      ]}
+    EOF
+    kafka-reassign-partitions --zookeeper {{ $zk }} --reassignment-json-file {{ $topic.name }}-increase-replication-factor.json --execute
+    kafka-reassign-partitions --zookeeper {{ $zk }} --reassignment-json-file {{ $topic.name }}-increase-replication-factor.json --verify
+    {{- else if and $topic.partitions $topic.replicationFactor }}
     kafka-topics --zookeeper {{ $zk }} --create --if-not-exists --force --topic {{ $topic.name }} --partitions {{ $topic.partitions }} --replication-factor {{ $topic.replicationFactor }}
     {{- else if $topic.partitions }}
     kafka-topics --zookeeper {{ $zk }} --alter --force --topic {{ $topic.name }} --partitions {{ $topic.partitions }} || true

incubator/kafka/values.yaml

@@ -311,16 +311,24 @@ prometheus:
 ## Topic creation and configuration.
 ## The job will be run on a deployment only when the config has been changed.
 ## - If 'partitions' and 'replicationFactor' are specified we create the topic (with --if-not-exists.)
+## - If 'partitions', 'replicationFactor' and 'reassignPartitions' are specified we reassign the partitions to
+## increase the replication factor of an existing topic.
 ## - If 'partitions' is specified we 'alter' the number of partitions. This will
 ## silently and safely fail if the new setting isn’t strictly larger than the old (i.e. a NOOP.) Do be aware of the
 ## implications for keyed topics (ref: https://docs.confluent.io/current/kafka/post-deployment.html#admin-operations)
 ## - If 'defaultConfig' is specified it's deleted from the topic configuration. If it isn't present,
 ## it will silently and safely fail.
 ## - If 'config' is specified it's added to the topic configuration.
 ##
+## Note: To increase the 'replicationFactor' of a topic, 'reassignPartitions' must be set to true (see above).
+##
 topics: []
   # - name: myExistingTopicConfig
   #   config: "cleanup.policy=compact,delete.retention.ms=604800000"
+  # - name: myExistingTopicReassignPartitions
+  #   partitions: 8
+  #   replicationFactor: 5
+  #   reassignPartitions: true
   # - name: myExistingTopicPartitions
   #   partitions: 8
   # - name: myNewTopicWithConfig