Skip to content

Commit 2d3a2f4

Browse files
baibai
committed
Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image
1 parent f214fdd commit 2d3a2f4

File tree

4 files changed

+84
-93
lines changed

4 files changed

+84
-93
lines changed

Diff for: ‎.github/workflows/ci.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
fail-fast: false
1515
matrix:
1616
go-version: [1.16.x, 1.17.x]
17-
kafka-version: [2.7.1, 2.8.1, 3.0.0]
17+
kafka-version: [3.0.0, 3.1.0]
1818
platform: [ubuntu-latest]
1919

2020
env:
@@ -48,7 +48,7 @@ jobs:
4848
4949
- name: Install dependencies
5050
run: |
51-
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1
51+
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.0
5252
export REPOSITORY_ROOT=${GITHUB_WORKSPACE}
5353
5454
- name: Run test suite

Diff for: ‎docker-compose.yml

+78-73
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
version: '3.7'
22
services:
33
zookeeper-1:
4-
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
4+
image: 'confluentinc/cp-zookeeper:7.0.1'
55
restart: always
66
environment:
77
ZOOKEEPER_SERVER_ID: '1'
@@ -13,7 +13,7 @@ services:
1313
ZOOKEEPER_SYNC_LIMIT: '5'
1414
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
1515
zookeeper-2:
16-
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
16+
image: 'confluentinc/cp-zookeeper:7.0.1'
1717
restart: always
1818
environment:
1919
ZOOKEEPER_SERVER_ID: '2'
@@ -25,7 +25,7 @@ services:
2525
ZOOKEEPER_SYNC_LIMIT: '5'
2626
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
2727
zookeeper-3:
28-
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
28+
image: 'confluentinc/cp-zookeeper:7.0.1'
2929
restart: always
3030
environment:
3131
ZOOKEEPER_SERVER_ID: '3'
@@ -37,90 +37,95 @@ services:
3737
ZOOKEEPER_SYNC_LIMIT: '5'
3838
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
3939
kafka-1:
40-
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
40+
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
4141
restart: always
4242
environment:
43-
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
44-
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
45-
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
46-
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
47-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
48-
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
49-
KAFKA_BROKER_ID: '1'
50-
KAFKA_BROKER_RACK: '1'
51-
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
52-
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
53-
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
54-
KAFKA_DELETE_TOPIC_ENABLE: 'true'
55-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
43+
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
44+
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
45+
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
46+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
47+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
48+
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
49+
KAFKA_CFG_BROKER_ID: '1'
50+
KAFKA_CFG_BROKER_RACK: '1'
51+
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
52+
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
53+
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
54+
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
55+
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
56+
ALLOW_PLAINTEXT_LISTENER: 'true'
5657
kafka-2:
57-
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
58+
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
5859
restart: always
5960
environment:
60-
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
61-
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
62-
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
63-
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
64-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
65-
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
66-
KAFKA_BROKER_ID: '2'
67-
KAFKA_BROKER_RACK: '2'
68-
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
69-
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
70-
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
71-
KAFKA_DELETE_TOPIC_ENABLE: 'true'
72-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
61+
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
62+
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
63+
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
64+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
65+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
66+
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
67+
KAFKA_CFG_BROKER_ID: '2'
68+
KAFKA_CFG_BROKER_RACK: '2'
69+
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
70+
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
71+
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
72+
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
73+
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
74+
ALLOW_PLAINTEXT_LISTENER: 'true'
7375
kafka-3:
74-
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
76+
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
7577
restart: always
7678
environment:
77-
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
78-
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
79-
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
80-
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
81-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
82-
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
83-
KAFKA_BROKER_ID: '3'
84-
KAFKA_BROKER_RACK: '3'
85-
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
86-
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
87-
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
88-
KAFKA_DELETE_TOPIC_ENABLE: 'true'
89-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
79+
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
80+
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
81+
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
82+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
83+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
84+
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
85+
KAFKA_CFG_BROKER_ID: '3'
86+
KAFKA_CFG_BROKER_RACK: '3'
87+
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
88+
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
89+
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
90+
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
91+
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
92+
ALLOW_PLAINTEXT_LISTENER: 'true'
9093
kafka-4:
91-
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
94+
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
9295
restart: always
9396
environment:
94-
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
95-
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
96-
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
97-
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
98-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
99-
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
100-
KAFKA_BROKER_ID: '4'
101-
KAFKA_BROKER_RACK: '4'
102-
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
103-
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
104-
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
105-
KAFKA_DELETE_TOPIC_ENABLE: 'true'
106-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
97+
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
98+
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
99+
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
100+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
101+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
102+
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
103+
KAFKA_CFG_BROKER_ID: '4'
104+
KAFKA_CFG_BROKER_RACK: '4'
105+
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
106+
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
107+
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
108+
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
109+
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
110+
ALLOW_PLAINTEXT_LISTENER: 'true'
107111
kafka-5:
108-
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
112+
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
109113
restart: always
110114
environment:
111-
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
112-
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
113-
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
114-
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
115-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
116-
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
117-
KAFKA_BROKER_ID: '5'
118-
KAFKA_BROKER_RACK: '5'
119-
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
120-
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
121-
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
122-
KAFKA_DELETE_TOPIC_ENABLE: 'true'
123-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
115+
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
116+
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
117+
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
118+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
119+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
120+
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
121+
KAFKA_CFG_BROKER_ID: '5'
122+
KAFKA_CFG_BROKER_RACK: '5'
123+
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
124+
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
125+
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
126+
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
127+
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
128+
ALLOW_PLAINTEXT_LISTENER: 'true'
124129
toxiproxy:
125130
image: 'ghcr.io/shopify/toxiproxy:2.3.0'
126131
ports:

Diff for: ‎functional_test.go

+3-18
Original file line numberDiff line numberDiff line change
@@ -136,28 +136,13 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
136136
if version, ok := os.LookupEnv("KAFKA_VERSION"); ok {
137137
env.KafkaVersion = version
138138
} else {
139-
// We have cp-7.0.0 as the default in the docker-compose file, so that's kafka 3.0.0.
140-
env.KafkaVersion = "3.0.0"
141-
}
142-
143-
// the mapping of confluent platform docker image versions -> kafka versions can be
144-
// found here: https://docs.confluent.io/current/installation/versions-interoperability.html
145-
var confluentPlatformVersion string
146-
switch env.KafkaVersion {
147-
case "3.0.0":
148-
confluentPlatformVersion = "7.0.1"
149-
case "2.8.1":
150-
confluentPlatformVersion = "6.2.2"
151-
case "2.7.1":
152-
confluentPlatformVersion = "6.1.4"
153-
default:
154-
return fmt.Errorf("don't know what confluent platform version to use for kafka %s", env.KafkaVersion)
139+
env.KafkaVersion = "3.1.0"
155140
}
156141

157142
c := exec.Command("docker-compose", "up", "-d")
158143
c.Stdout = os.Stdout
159144
c.Stderr = os.Stderr
160-
c.Env = append(os.Environ(), fmt.Sprintf("CONFLUENT_PLATFORM_VERSION=%s", confluentPlatformVersion))
145+
c.Env = append(os.Environ(), fmt.Sprintf("KAFKA_VERSION=%s", env.KafkaVersion))
161146
err := c.Run()
162147
if err != nil {
163148
return fmt.Errorf("failed to run docker-compose to start test environment: %w", err)
@@ -169,7 +154,7 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
169154

170155
// Wait for the kafka broker to come up
171156
allBrokersUp := false
172-
for i := 0; i < 45 && !allBrokersUp; i++ {
157+
for i := 0; i < 90 && !allBrokersUp; i++ {
173158
Logger.Println("waiting for kafka brokers to come up")
174159
time.Sleep(1 * time.Second)
175160
config := NewTestConfig()

Diff for: ‎utils_test.go

+1
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ func TestVersionParsing(t *testing.T) {
6969
"2.8.0",
7070
"2.8.1",
7171
"3.0.0",
72+
"3.1.0",
7273
}
7374
for _, s := range validVersions {
7475
v, err := ParseKafkaVersion(s)

0 commit comments

Comments
 (0)