Skip to content

Commit 752cd45

Browse files
solssonsolsson
authored
Merge pull request #47 from Yolean/config-init
Use init container for per-pod config
2 parents 6f8f6d4 + b848f85 commit 752cd45

File tree

5 files changed

+70
-34
lines changed

5 files changed

+70
-34
lines changed

‎10broker-config.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ metadata:
44
namespace: kafka
55
apiVersion: v1
66
data:
7+
init.sh: |-
8+
#!/bin/bash
9+
set -x
10+
11+
export KAFKA_BROKER_ID=${HOSTNAME##*-}
12+
sed -i "s/\${KAFKA_BROKER_ID}/$KAFKA_BROKER_ID/" /etc/kafka/server.properties
13+
714
server.properties: |-
815
# Licensed to the Apache Software Foundation (ASF) under one or more
916
# contributor license agreements. See the NOTICE file distributed with
@@ -25,7 +32,7 @@ data:
2532
############################# Server Basics #############################
2633
2734
# The id of the broker. This must be set to a unique integer for each broker.
28-
broker.id=0
35+
broker.id=${KAFKA_BROKER_ID}
2936
3037
# Switch to enable topic deletion or not, default value is false
3138
#delete.topic.enable=true

‎50kafka.yml‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ spec:
1515
prometheus.io/port: "5556"
1616
spec:
1717
terminationGracePeriodSeconds: 30
18+
initContainers:
19+
- name: init-config
20+
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
21+
command: ['/bin/bash', '/etc/kafka/init.sh']
22+
volumeMounts:
23+
- name: config
24+
mountPath: /etc/kafka
1825
containers:
1926
- name: metrics
2027
image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
@@ -31,22 +38,22 @@ spec:
3138
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
3239
env:
3340
- name: KAFKA_LOG4J_OPTS
34-
value: -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties
41+
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
3542
- name: JMX_PORT
3643
value: "5555"
3744
ports:
3845
- containerPort: 9092
3946
command:
40-
- /bin/sh
41-
- -c
42-
- >
43-
./bin/kafka-server-start.sh
44-
./config/server.properties
45-
--override zookeeper.connect=zookeeper:2181
46-
--override log.retention.hours=-1
47-
--override log.dirs=/var/lib/kafka/data/topics
48-
--override broker.id=${HOSTNAME##*-}
49-
--override auto.create.topics.enable=false
47+
- ./bin/kafka-server-start.sh
48+
- /etc/kafka/server.properties
49+
- --override
50+
- zookeeper.connect=zookeeper:2181
51+
- --override
52+
- log.retention.hours=-1
53+
- --override
54+
- log.dirs=/var/lib/kafka/data/topics
55+
- --override
56+
- auto.create.topics.enable=false
5057
resources:
5158
requests:
5259
cpu: 100m
@@ -59,7 +66,7 @@ spec:
5966
- 'echo "" | nc -w 1 127.0.0.1 9092'
6067
volumeMounts:
6168
- name: config
62-
mountPath: /opt/kafka/config
69+
mountPath: /etc/kafka
6370
- name: data
6471
mountPath: /var/lib/kafka/data
6572
volumes:

‎zookeeper/10zookeeper-config.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@ metadata:
44
namespace: kafka
55
apiVersion: v1
66
data:
7+
init.sh: |-
8+
#!/bin/bash
9+
set -x
10+
11+
[ -z "$ID_OFFSET" ] && ID_OFFSET=1
12+
export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET))
13+
echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid
14+
sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties
15+
716
zookeeper.properties: |-
817
tickTime=2000
918
dataDir=/var/lib/zookeeper/data
@@ -16,9 +25,13 @@ data:
1625
server.3=pzoo-2.pzoo:2888:3888:participant
1726
server.4=zoo-0.zoo:2888:3888:participant
1827
server.5=zoo-1.zoo:2888:3888:participant
19-
28+
2029
log4j.properties: |-
2130
log4j.rootLogger=INFO, stdout
2231
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
2332
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
2433
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
34+
35+
# Suppress connection log messages, three lines per livenessProbe execution
36+
log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN
37+
log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN

‎zookeeper/50pzoo.yml‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@ spec:
1616
prometheus.io/port: "5556"
1717
spec:
1818
terminationGracePeriodSeconds: 10
19+
initContainers:
20+
- name: init-config
21+
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
22+
command: ['/bin/bash', '/etc/kafka/init.sh']
23+
volumeMounts:
24+
- name: config
25+
mountPath: /etc/kafka
26+
- name: data
27+
mountPath: /var/lib/zookeeper/data
1928
containers:
2029
- name: metrics
2130
image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
@@ -40,18 +49,12 @@ spec:
4049
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
4150
env:
4251
- name: KAFKA_LOG4J_OPTS
43-
value: -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties
52+
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
4453
- name: JMX_PORT
4554
value: "5555"
4655
command:
47-
- /bin/sh
48-
- -euc
49-
- >
50-
export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 1));
51-
echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid;
52-
sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" config/zookeeper.properties;
53-
cat config/zookeeper.properties;
54-
./bin/zookeeper-server-start.sh config/zookeeper.properties
56+
- ./bin/zookeeper-server-start.sh
57+
- /etc/kafka/zookeeper.properties
5558
ports:
5659
- containerPort: 2181
5760
name: client
@@ -77,7 +80,7 @@ spec:
7780
- '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]'
7881
volumeMounts:
7982
- name: config
80-
mountPath: /opt/kafka/config
83+
mountPath: /etc/kafka
8184
- name: data
8285
mountPath: /var/lib/zookeeper/data
8386
volumes:

‎zookeeper/51zoo.yml‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,18 @@ spec:
1616
prometheus.io/port: "5556"
1717
spec:
1818
terminationGracePeriodSeconds: 10
19+
initContainers:
20+
- name: init-config
21+
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
22+
command: ['/bin/bash', '/etc/kafka/init.sh']
23+
env:
24+
- name: ID_OFFSET
25+
value: "4"
26+
volumeMounts:
27+
- name: config
28+
mountPath: /etc/kafka
29+
- name: data
30+
mountPath: /var/lib/zookeeper/data
1931
containers:
2032
- name: metrics
2133
image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
@@ -40,18 +52,12 @@ spec:
4052
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
4153
env:
4254
- name: KAFKA_LOG4J_OPTS
43-
value: -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties
55+
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
4456
- name: JMX_PORT
4557
value: "5555"
4658
command:
47-
- /bin/sh
48-
- -euc
49-
- >
50-
export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 4));
51-
echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid;
52-
sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" config/zookeeper.properties;
53-
cat config/zookeeper.properties;
54-
./bin/zookeeper-server-start.sh config/zookeeper.properties
59+
- ./bin/zookeeper-server-start.sh
60+
- /etc/kafka/zookeeper.properties
5561
ports:
5662
- containerPort: 2181
5763
name: client
@@ -77,7 +83,7 @@ spec:
7783
- '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]'
7884
volumeMounts:
7985
- name: config
80-
mountPath: /opt/kafka/config
86+
mountPath: /etc/kafka
8187
- name: data
8288
mountPath: /var/lib/zookeeper/data
8389
volumes:

0 commit comments

Comments
 (0)