-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathdocker-compose-tls.yml
201 lines (188 loc) · 8.57 KB
/
docker-compose-tls.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
version: '3.6'
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.0.1
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
- "2182:2182"
- "8080:8080"
environment:
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_SERVERS: zookeeper:22888:23888
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_SECURE_CLIENT_PORT: 2182
ZOOKEEPER_AUTH_PROVIDER_X509: "org.apache.zookeeper.server.auth.X509AuthenticationProvider"
ZOOKEEPER_SERVER_CNXN_FACTORY: "org.apache.zookeeper.server.NettyServerCnxnFactory"
ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /etc/zookeeper/secrets/kafka.zookeeper.truststore.jks
ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: $ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD
ZOOKEEPER_SSL_KEYSTORE_LOCATION: /etc/zookeeper/secrets/kafka.zookeeper.keystore.jks
ZOOKEEPER_SSL_KEYSTORE_PASSWORD: $ZOOKEEPER_SSL_KEYSTORE_PASSWORD
ZOOKEEPER_SSL_CLIENT_AUTH: 'need'
volumes:
- $PWD/secrets/zookeeper:/etc/zookeeper/secrets
broker1:
image: confluentinc/cp-kafka:7.0.1
hostname: broker1
container_name: broker1
depends_on:
- zookeeper
ports:
- "19092:19092"
- "19093:19093"
environment:
KAFKA_ADVERTISED_HOST_NAME: $BROKER_KAFKA_ADVERTISED_HOST_NAME
KAFKA_BROKER_ID: 1
KAFKA_LISTENERS: PLAINTEXT://:19092,SSL://:19093
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:19092,SSL://broker1:19093
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SSL:SSL
# Environment variables for SSL Zookeeper security between zookeeper and broker
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2182'
KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET: "org.apache.zookeeper.ClientCnxnSocketNetty"
KAFKA_ZOOKEEPER_SSL_CLIENT_ENABLE: "true"
KAFKA_ZOOKEEPER_SSL_PROTOCOL: TLSv1.3
KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.broker.truststore.jks
KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: $BROKER_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD
KAFKA_ZOOKEEPER_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.broker.keystore.jks
KAFKA_ZOOKEEPER_SSL_KEYSTORE_PASSWORD: $BROKER_ZOOKEEPER_SSL_KEYSTORE_PASSWORD
KAFKA_ZOOKEEPER_SET_ACL: "true"
# Environment variables for SSL Kafka security between broker and its clients
KAFKA_SSL_KEYSTORE_FILENAME: kafka.broker1.keystore.jks
KAFKA_SSL_KEYSTORE_CREDENTIALS: broker1_keystore_creds
KAFKA_SSL_KEY_CREDENTIALS: broker1_sslkey_creds
KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.broker1.truststore.jks
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: broker1_truststore_creds
KAFKA_SSL_ENABLED_PROTOCOLS: TLSv1.3
KAFKA_SECURITY_INTER_BROKER_PROTOCOL: SSL
KAFKA_SSL_CLIENT_AUTH: 'requested'
KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ''
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SSL:SSL,PLAINTEXT:PLAINTEXT
volumes:
- $PWD/secrets/broker1:/etc/kafka/secrets
connect:
image: confluentinc/cp-kafka-connect:7.0.1
hostname: connect
container_name: connect
depends_on:
- zookeeper
- broker1
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: broker1:19093
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'false'
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'false'
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_PLUGIN_PATH: /usr/share/confluent-hub-components
CONNECT_LOG4J_ROOT_LOGLEVEL: "FATAL"
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=FATAL,org.reflections=FATAL,com.mongodb.kafka=FATAL"
CONNECT_SECURITY_PROTOCOL: SSL
CONNECT_SSL_TRUSTSTORE_LOCATION: /etc/kafka-connect/secrets/kafka.connect.truststore.jks
CONNECT_SSL_TRUSTSTORE_PASSWORD: $CONNECT_SSL_TRUSTSTORE_PASSWORD
CONNECT_SSL_KEYSTORE_LOCATION: /etc/kafka-connect/secrets/kafka.connect.keystore.jks
CONNECT_SSL_KEYSTORE_PASSWORD: $CONNECT_SSL_KEYSTORE_PASSWORD
CONNECT_CONSUMER_BOOTSTRAP_SERVERS: 'broker1:19093'
CONNECT_CONSUMER_SECURITY_PROTOCOL: SSL
CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION: /etc/kafka-connect/secrets/kafka.connect.truststore.jks
CONNECT_CONSUMER_SSL_TRUSTSTORE_PASSWORD: $CONNECT_SSL_TRUSTSTORE_PASSWORD
CONNECT_PRODUCER_BOOTSTRAP_SERVERS: 'broker1:19093'
CONNECT_PRODUCER_SECURITY_PROTOCOL: SSL
CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION: /etc/kafka-connect/secrets/kafka.connect.truststore.jks
CONNECT_PRODUCER_SSL_TRUSTSTORE_PASSWORD: $CONNECT_SSL_TRUSTSTORE_PASSWORD
#KAFKA_OPTS should be avoided for Atlas MongoDB connections
KAFKA_OPTS: "-Djavax.net.ssl.trustStore=/etc/kafka-connect/secrets/kafka.connect.truststore.jks -Djavax.net.ssl.trustStorePassword=$CONNECT_SSL_TRUSTSTORE_PASSWORD -Djavax.net.ssl.keyStore=/etc/kafka-connect/secrets/kafka.connect.keystore.jks -Djavax.net.ssl.keyStorePassword=$CONNECT_SSL_KEYSTORE_PASSWORD"
volumes:
- $PWD/mongodb-kafka-connect:/usr/share/confluent-hub-components/kafka-connect-mongodb
- $PWD/kafka/scripts:/scripts
- $PWD/secrets/connect:/etc/kafka-connect/secrets
command:
- bash
- -c
- |
sleep 5
echo "Launching Kafka Connect worker"
/etc/confluent/docker/run &
#
echo "Waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME"
while [ $$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors) -ne 200 ]; do
echo -e $$(date) "Kafka Connect listener HTTP state: "$$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors)" (waiting for 200)"
sleep 5
done
nc -vz $$CONNECT_REST_ADVERTISED_HOST_NAME $$CONNECT_REST_PORT
echo -e "\n--\n+> Creating Kafka Connect MongoDB sink"
chmod 755 /scripts/sink-connect.sh
/scripts/sink-connect.sh
sleep infinity
producer:
#image: librdkafka:1.7.0
build:
context: ./librdkafka-producer/.
dockerfile: Dockerfile
depends_on:
- zookeeper
- broker1
- connect
- mongo
hostname: producer
container_name: producer
privileged: true
volumes:
- $PWD/librdkafka-producer/conf/ld.so.conf:/etc/ld.so.conf
- $PWD/librdkafka-producer/src:/opt/producer
- /var/run/docker.sock:/host/var/run/docker.sock
- /dev:/host/dev
- /proc:/host/proc:ro
- /boot:/host/boot:ro
- /lib/modules:/host/lib/modules:ro
- /usr:/host/usr:ro
- $PWD/secrets/producer:/opt/producer/secrets
working_dir: /opt/producer
command:
- bash
- -c
- |
echo "Launching librdkafka producer"
ldconfig
/producer_entrypoint.sh &
echo Waiting for kafka service start...;
while [ $$(timeout 1 bash -c 'cat < /dev/null > /dev/tcp/broker1/19093' 2&>1; echo $$?) -ne 0 ];do
sleep 1;
done;
echo Connected!;
echo -e "\n--\n+> Starting kafka producer"
/opt/producer/producer broker1:19093 sysdigtopic
sleep infinity
mongo:
image: mongo:5.0.5
hostname: mongo
container_name: mongo
restart: always
environment:
MONGO_INITDB_ROOT_USERNAME: $MONGO_ROOT_USER
MONGO_INITDB_ROOT_PASSWORD: $MONGO_ROOT_PASSWORD
ports:
- 27017:27017
user: "${UID}:${GID}"
volumes:
- $PWD/mongodb/conf/mongod.conf:/etc/mongod.conf
- $PWD/mongodb/initdb.d/:/docker-entrypoint-initdb.d/
- $PWD/mongodb/data/db/:/data/db/
- $PWD/secrets/mongo:/etc/ssl
command: [ "-f", "/etc/mongod.conf" ]