-
Notifications
You must be signed in to change notification settings - Fork 13
/
Copy pathkafka-tests.yml
246 lines (238 loc) · 14 KB
/
kafka-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
- name: coyote
title: Fast Data CSD Tests
# Tests we should add:
# - kafka-avro-console-consumer
# - test hdfs-sink (difficult, test with kerberos in standalone)
- name: Brokers
skip: _brokers_
entries:
# BASIC / PERFORMANCE
- name: Create Topic (basic kafka)
command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote_test_01 --partition 3 --replication 1 --create
- name: List Topics (basic kafka)
command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --list
- name: Performance Test (basic kafka)
command: |
kafka-producer-perf-test --topic coyote_test_01 --throughput 100000 --record-size 1000 --num-records 2000000
--producer-props bootstrap.servers="cloudera.landoop.com:9092"
timeout: 90s
- name: REST Proxy
skip: _rest_proxy_
entries:
# HTTP REST PROXY
- name: List Topics (rest proxy)
command: curl -vs --stderr - "http://cloudera.landoop.com:8082/topics"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Topic Information (rest proxy)
command: curl -vs --stderr - "http://cloudera.landoop.com:8082/topics/coyote_test_01"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Topic Partitions (rest proxy)
command: curl -vs --stderr - "http://cloudera.landoop.com:8082/topics/coyote_test_01/partitions"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Delete Topic (basic kafka)
command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote_test_01 --delete
- name: Produce Avro Message (rest proxy, schema registry)
# Please do not change the formatting (e.g. add new lines) of the JSON message below, REST Proxy is very sensitive.
command: |
curl -vs --stderr - -XPOST
-H "Content-Type: application/vnd.kafka.avro.v1+json"
--data '{"value_schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}",
"records": [{"value": {"name": "testUser"}}]}'
"http://cloudera.landoop.com:8082/topics/coyote_test_avro"
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Create Consumer for Avro data (rest proxy, schema registry)
command: |
curl -vs --stderr - -XPOST -H "Content-Type: application/vnd.kafka.v1+json"
--data '{"name": "a_consumer", "format": "avro", "auto.offset.reset": "smallest"}'
"http://cloudera.landoop.com:8082/consumers/coyote_avro"
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Consume Avro Message (rest proxy, schema registry)
command: |
curl -vs --stderr - -XGET -H "Accept: application/vnd.kafka.avro.v1+json"
"http://cloudera.landoop.com:8082/consumers/coyote_avro/instances/a_consumer/topics/coyote_test_avro"
stdout_has: [ 'testUser' ]
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Delete Avro Consumer (rest proxy, schema registry)
command: curl -vs --stderr - -X DELETE "http://cloudera.landoop.com:8082/consumers/coyote_avro/instances/a_consumer"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote_test_avro --delete
nolog: true
- name: Produce JSON Message (rest proxy)
command: |
curl -vs --stderr - -XPOST -H "Content-Type: application/vnd.kafka.json.v1+json"
--data '{"records":[{"value":{"foo":"bar"}}]}' "http://cloudera.landoop.com:8082/topics/coyote_test_json"
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Create Consumer for JSON data (rest proxy)
command: |
curl -vs --stderr - -XPOST -H "Content-Type: application/vnd.kafka.v1+json"
--data '{"name": "a_consumer", "format": "json", "auto.offset.reset": "smallest"}'
"http://cloudera.landoop.com:8082/consumers/coyote_json"
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Consume JSON Message (rest proxy)
command: |
curl -vs --stderr - -XGET -H "Accept: application/vnd.kafka.json.v1+json" \
"http://cloudera.landoop.com:8082/consumers/coyote_json/instances/a_consumer/topics/coyote_test_json"
stdout_has: [ 'foo.*bar' ]
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Delete JSON Consumer (rest proxy)
command: curl -vs --stderr - -X DELETE "http://cloudera.landoop.com:8082/consumers/coyote_json/instances/a_consumer"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote_test_json --delete
nolog: true
- name: Produce Binary Message (rest proxy)
command: |
curl -vs --stderr - -XPOST -H "Content-Type: application/vnd.kafka.binary.v1+json"
--data '{"records":[{"value":"S2Fma2E="}]}' "http://cloudera.landoop.com:8082/topics/coyote_test_binary"
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Create Consumer for Binary data (rest proxy)
command: |
curl -vs --stderr - -XPOST -H "Content-Type: application/vnd.kafka.v1+json"
--data '{"name": "a_consumer", "format": "binary", "auto.offset.reset": "smallest"}'
"http://cloudera.landoop.com:8082/consumers/coyote_binary"
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Consume Binary Message (rest proxy)
command: |
curl -vs --stderr - -XGET -H "Accept: application/vnd.kafka.binary.v1+json" \
"http://cloudera.landoop.com:8082/consumers/coyote_binary/instances/a_consumer/topics/coyote_test_binary"
stdout_has: [ 'S2Fma2E=' ]
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Delete Binary Consumer (rest proxy)
command: curl -vs --stderr - -XDELETE "http://cloudera.landoop.com:8082/consumers/coyote_binary/instances/a_consumer"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote_test_binary --delete
nolog: true
- name: Schema Registry
skip: _schema_registry_
entries:
# SCHEMA REGISTRY
- name: Register a new Schema version (schema registry)
command: |
curl -vs --stderr - -XPOST -i -H "Content-Type: application/vnd.schemaregistry.v1+json"
--data '{"schema": "{\"type\": \"string\"}"}'
"http://cloudera.landoop.com:8081/subjects/coyote_test_01/versions"
- name: List subjects (schema registry)
command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8081/subjects"
stdout_has: [ 'coyote_test_01' ]
- name: List Schema versions (schema registry)
command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8081/subjects/coyote_test_01/versions"
# - name: Fetch Schema by globally unique id 1 (schema registry)
# command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8081/schemas/ids/1"
# stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Fetch Schema by name and version (schema registry)
command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8081/subjects/coyote_test_01/versions/1"
stdout_has: [ '"subject":"coyote_test_01","version":1' ]
stdout_not_has: [ 'error_code":[0-9]', 'Unexpected', 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Register Complex Schema (schema registry)
command: |
curl -vs --stderr - -XPOST -i -H "Content-Type: application/vnd.schemaregistry.v1+json"
--data '{"schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}"}'
"http://cloudera.landoop.com:8081/subjects/coyote_test_02/versions"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Test Schema Compatibility (schema registry)
command: |
curl -vs --stderr - -XPOST -i -H "Content-Type: application/vnd.schemaregistry.v1+json"
--data '{"schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}, {\"name\": \"address\", \"type\": \"string\"}]}"}'
"http://cloudera.landoop.com:8081/compatibility/subjects/coyote_test_02/versions/latest"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
stdout_has: [ 'is_compatible' ]
- name: Get Schema Registry Configuration (schema registry)
command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8081/config"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Connect
skip: _connect_
entries:
# CONNECT
- name: Get list of Connectors (connect distributed)
command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8085/connectors"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Create a Console Connector (connect distributed)
command: |
curl -vs --stderr - -X POST -H "Content-Type: application/json"
--data '{ "name": "coyote_test_console_source-%UNIQUE_CD%",
"config": {"connector.class":"org.apache.kafka.connect.file.FileStreamSourceConnector","tasks.max":"1","topic":"coyote_connect_distributed_test-%UNIQUE_CD%","file":"/etc/fstab"}}'
"http://cloudera.landoop.com:8085/connectors"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Get Connector s Configuration (connect distributed)
command: curl -vs --stderr - -XGET -i "http://cloudera.landoop.com:8085/connectors/coyote_test_console_source-%UNIQUE_CD%"
stdout_has: [ '/etc/fstab' ]
- name: kafconcli (kafconcli, connect distributed)
command: kafconcli get -e "http://cloudera.landoop.com:8085" coyote_test_console_source-%UNIQUE_CD%
stdout_has: [ '/etc/fstab' ]
- name: Sleep a bit to let the connector work
command: sleep 9
nolog: true
- name: "Run Console Consumer to fix Kafka's transient state (basic kafka)"
command: |
timeout 5
kafka-console-consumer --new-consumer
--bootstrap-server cloudera.landoop.com:9092
--topic coyote_connect_distributed_test-%UNIQUE_CD%
--from-beginning
ignore_exit_code: true
- name: Run Console Consumer (basic kafka)
command: |
timeout 5
kafka-console-consumer --new-consumer
--bootstrap-server cloudera.landoop.com:9092
--topic coyote_connect_distributed_test-%UNIQUE_CD%
--from-beginning
stdout_has: [ '/etc/fstab' ]
ignore_exit_code: true
- name: Delete connector
command: curl -vs --stderr - -XDELETE "http://cloudera.landoop.com:8085/connectors/coyote_test_console_source-%UNIQUE_CD%"
stdout_not_has: [ 'HTTP/1.1 [45][0-9][0-9] ' ]
- name: Delete Connect Distributes Test Topic (basic kafka)
command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote_connect_distributed_test-%UNIQUE_CD% --delete
- command: rm -rf coyote_test.sqlite coyote_sqlite_connector.properties coyote_connect_standalone.properties coyote_connect.offset
nolog: true
- name: Create and Init SQLite database
command: sqlite3 coyote_test.sqlite
stdin: |
CREATE TABLE accounts(id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255));
INSERT INTO accounts(name) VALUES('alice');
INSERT INTO accounts(name) VALUES('bob');
- name: Create coyote_sqlite_connector.properties
command: tee coyote_sqlite_connector.properties
stdin: |
name=coyote-test-connect-standalone-%UNIQUE%
connector.class=io.confluent.connect.jdbc.JdbcSourceConnector
tasks.max=1
connection.url=jdbc:sqlite:coyote_test.sqlite
mode=incrementing
incrementing.column.name=id
topic.prefix=coyote-test-connect-standalone-
- name: Create coyote_connect_standalone.properties
command: tee coyote_connect_standalone.properties
stdin: |
bootstrap.servers=cloudera.landoop.com:9092
key.converter=io.confluent.connect.avro.AvroConverter
key.converter.schema.registry.url=http://cloudera.landoop.com:8081
value.converter=io.confluent.connect.avro.AvroConverter
value.converter.schema.registry.url=http://cloudera.landoop.com:8081
internal.key.converter=org.apache.kafka.connect.json.JsonConverter
internal.value.converter=org.apache.kafka.connect.json.JsonConverter
internal.key.converter.schemas.enable=false
internal.value.converter.schemas.enable=false
offset.storage.file.filename=coyote_connect.offset
offset.flush.interval.ms=5000
zookeeper=cloudera.landoop.com:2181/fastdata
rest.port=38081
port=38081
kafka.logs.dir=logs/
- name: Read SQLite into Topic (connect standalone)
command: timeout 9 connect-standalone coyote_connect_standalone.properties coyote_sqlite_connector.properties
stdout_not_has: [ 'ERROR' ]
ignore_exit_code: true
- name: Run Console Consumer (basic kafka)
command: |
timeout 5
kafka-console-consumer --new-consumer
--bootstrap-server cloudera.landoop.com:9092
--topic coyote-test-connect-standalone-accounts
--from-beginning
stdout_has: [ 'alice', 'bob' ]
ignore_exit_code: true
- name: Delete Connect Standalone Test Topic (basic kafka)
command: kafka-topics --zookeeper cloudera.landoop.com:2181/fastdata --topic coyote-test-connect-standalone-accounts --delete
- command: rm -rf coyote_test.sqlite coyote_sqlite_connector.properties coyote_connect_standalone.properties coyote_connect.offset
nolog: true