Skip to content

Commit a1fdbd6

Browse files
author
Hendrik Muhs
authored
[Transform] Add yml test suite for testing remote clusters (CCS) (#51033)
add a test suite for remote clusters features and add test cases for transform
1 parent 7978f0b commit a1fdbd6

File tree

4 files changed

+496
-0
lines changed

4 files changed

+496
-0
lines changed

‎x-pack/qa/multi-cluster-tests-with-security/build.gradle‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import org.elasticsearch.gradle.test.RestIntegTestTask
2+
3+
apply plugin: 'elasticsearch.testclusters'
4+
apply plugin: 'elasticsearch.standalone-test'
5+
6+
dependencies {
7+
testCompile project(':x-pack:qa')
8+
testCompile project(':client:rest-high-level')
9+
}
10+
11+
task 'remote-cluster'(type: RestIntegTestTask) {
12+
mustRunAfter(precommit)
13+
runner {
14+
systemProperty 'tests.rest.suite', 'remote_cluster'
15+
}
16+
}
17+
18+
testClusters.'remote-cluster' {
19+
testDistribution = 'DEFAULT'
20+
numberOfNodes = 2
21+
setting 'cluster.remote.connect', "false"
22+
setting 'xpack.ilm.enabled', 'false'
23+
setting 'xpack.security.enabled', 'true'
24+
setting 'xpack.watcher.enabled', 'false'
25+
setting 'xpack.monitoring.enabled', 'false'
26+
setting 'xpack.license.self_generated.type', 'trial'
27+
28+
user username: "test_user", password: "x-pack-test-password"
29+
}
30+
31+
task 'mixed-cluster'(type: RestIntegTestTask) {
32+
dependsOn 'remote-cluster'
33+
runner {
34+
useCluster testClusters.'remote-cluster'
35+
systemProperty 'tests.rest.suite', 'multi_cluster'
36+
}
37+
}
38+
39+
testClusters.'mixed-cluster' {
40+
testDistribution = 'DEFAULT'
41+
setting 'xpack.security.enabled', 'true'
42+
setting 'xpack.watcher.enabled', 'false'
43+
setting 'xpack.monitoring.enabled', 'false'
44+
setting 'xpack.license.self_generated.type', 'trial'
45+
setting 'cluster.remote.my_remote_cluster.seeds', {
46+
testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
47+
}
48+
setting 'cluster.remote.connections_per_cluster', "1"
49+
setting 'cluster.remote.connect', "true"
50+
51+
user username: "test_user", password: "x-pack-test-password"
52+
}
53+
54+
task integTest {
55+
dependsOn 'mixed-cluster'
56+
}
57+
58+
test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
59+
check.dependsOn(integTest)

‎x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.multi_cluster;
8+
9+
import com.carrotsearch.randomizedtesting.annotations.Name;
10+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
11+
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
12+
13+
import org.apache.lucene.util.TimeUnits;
14+
import org.elasticsearch.common.settings.SecureString;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.common.util.concurrent.ThreadContext;
17+
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
18+
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
19+
20+
import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
21+
22+
@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
23+
public class MultiClusterYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
24+
25+
private static final String USER = "test_user";
26+
private static final String PASS = "x-pack-test-password";
27+
28+
@Override
29+
protected boolean preserveIndicesUponCompletion() {
30+
return true;
31+
}
32+
33+
public MultiClusterYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
34+
super(testCandidate);
35+
}
36+
37+
@ParametersFactory
38+
public static Iterable<Object[]> parameters() throws Exception {
39+
return createParameters();
40+
}
41+
42+
@Override
43+
protected Settings restClientSettings() {
44+
String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
45+
return Settings.builder()
46+
.put(super.restClientSettings())
47+
.put(ThreadContext.PREFIX + ".Authorization", token)
48+
.build();
49+
}
50+
}

‎x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
---
2+
setup:
3+
- skip:
4+
features: headers
5+
6+
- do:
7+
cluster.health:
8+
wait_for_status: yellow
9+
10+
- do:
11+
security.put_user:
12+
username: "joe"
13+
body: >
14+
{
15+
"password": "transform",
16+
"roles" : [ "transform_admin", "x_cluster_role" ]
17+
}
18+
- do:
19+
security.put_role:
20+
name: "x_cluster_role"
21+
body: >
22+
{
23+
"cluster": [],
24+
"indices": [
25+
{
26+
"names": ["test_index", "my_remote_cluster:test_i*", "my_remote_cluster:aliased_test_index"],
27+
"privileges": ["all", "view_index_metadata"]
28+
},
29+
{
30+
"names": ["simple-remote-transform", "simple-local-remote-transform"],
31+
"privileges": ["all"]
32+
}
33+
]
34+
}
35+
---
36+
teardown:
37+
- do:
38+
security.delete_user:
39+
username: "joe"
40+
ignore: 404
41+
42+
---
43+
"Search remote cluster":
44+
- do:
45+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
46+
search:
47+
rest_total_hits_as_int: true
48+
index: my_remote_cluster:test_index
49+
body:
50+
aggs:
51+
user:
52+
terms:
53+
field: user
54+
55+
- match: { _shards.total: 3 }
56+
- match: { hits.total: 9 }
57+
- length: { aggregations.user.buckets: 3 }
58+
- match: { aggregations.user.buckets.0.key: "a" }
59+
- match: { aggregations.user.buckets.0.doc_count: 5 }
60+
61+
---
62+
"Batch transform from remote cluster":
63+
- do:
64+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
65+
transform.put_transform:
66+
transform_id: "simple-remote-transform"
67+
body: >
68+
{
69+
"source": { "index": "my_remote_cluster:test_index" },
70+
"dest": { "index": "simple-remote-transform" },
71+
"pivot": {
72+
"group_by": { "user": {"terms": {"field": "user"}}},
73+
"aggs": {"avg_stars": {"avg": {"field": "stars"}}}
74+
}
75+
}
76+
- match: { acknowledged: true }
77+
78+
- do:
79+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
80+
transform.start_transform:
81+
transform_id: "simple-remote-transform"
82+
- match: { acknowledged: true }
83+
84+
- do:
85+
transform.get_transform_stats:
86+
transform_id: "simple-remote-transform"
87+
- match: { count: 1 }
88+
- match: { transforms.0.id: "simple-remote-transform" }
89+
- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
90+
91+
- do:
92+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
93+
transform.stop_transform:
94+
transform_id: "simple-remote-transform"
95+
wait_for_completion: true
96+
wait_for_checkpoint: true
97+
- match: { acknowledged: true }
98+
99+
- do:
100+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
101+
transform.get_transform_stats:
102+
transform_id: "simple-remote-transform"
103+
- match: { count: 1 }
104+
- match: { transforms.0.id: "simple-remote-transform" }
105+
- match: { transforms.0.state: "stopped" }
106+
- match: { transforms.0.checkpointing.last.checkpoint: 1 }
107+
108+
# workaround: refresh dest index, to be removed, see gh #51154
109+
- do:
110+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
111+
indices.refresh:
112+
index: simple-remote-transform
113+
114+
- do:
115+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
116+
search:
117+
rest_total_hits_as_int: true
118+
index: simple-remote-transform
119+
sort: user
120+
121+
- match: { hits.total: 3 }
122+
- match: { hits.hits.0._index: simple-remote-transform }
123+
- match: { hits.hits.0._source.avg_stars: 3.6 }
124+
- match: { hits.hits.0._source.user: a }
125+
- match: { hits.hits.1._source.avg_stars: 2.0 }
126+
- match: { hits.hits.1._source.user: b }
127+
128+
- do:
129+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
130+
transform.update_transform:
131+
transform_id: "simple-remote-transform"
132+
body: >
133+
{
134+
"source": { "index": ["my_remote_cluster:test_index", "my_remote_cluster:test_index_2"] }
135+
}
136+
- do:
137+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
138+
transform.get_transform_stats:
139+
transform_id: "simple-remote-transform"
140+
- match: { count: 1 }
141+
- match: { transforms.0.id: "simple-remote-transform" }
142+
- match: { transforms.0.state: "stopped" }
143+
# we added test_index_2, which has 2 more docs:
144+
- match: { transforms.0.checkpointing.operations_behind: 2 }
145+
146+
---
147+
"Batch transform from local and remote cluster":
148+
- do:
149+
indices.create:
150+
index: test_index
151+
body:
152+
settings:
153+
index:
154+
number_of_shards: 3
155+
number_of_replicas: 0
156+
aliases:
157+
test_alias: {}
158+
mappings:
159+
properties:
160+
time:
161+
type: date
162+
user:
163+
type: keyword
164+
stars:
165+
type: integer
166+
coolness:
167+
type: integer
168+
169+
- do:
170+
bulk:
171+
refresh: true
172+
body:
173+
- '{"index": {"_index": "test_index"}}'
174+
- '{"user": "a", "stars": 3, "date" : "2018-11-29T12:12:12.123456789Z"}'
175+
- '{"index": {"_index": "test_index"}}'
176+
- '{"user": "c", "stars": 5, "date" : "2018-11-29T12:14:12.123456789Z"}'
177+
- '{"index": {"_index": "test_index"}}'
178+
- '{"user": "d", "stars": 5, "date" : "2018-11-29T12:16:12.123456789Z"}'
179+
- '{"index": {"_index": "test_index"}}'
180+
- '{"user": "e", "stars": 2, "date" : "2018-11-29T12:17:12.123456789Z"}'
181+
- '{"index": {"_index": "test_index"}}'
182+
- '{"user": "b", "stars": 3, "date" : "2018-11-29T12:22:12.123456789Z"}'
183+
- '{"index": {"_index": "test_index"}}'
184+
- '{"user": "c", "stars": 5, "date" : "2018-11-29T12:23:12.123456789Z"}'
185+
- '{"index": {"_index": "test_index"}}'
186+
- '{"user": "d", "stars": 1, "date" : "2018-11-29T12:32:12.123456789Z"}'
187+
- '{"index": {"_index": "test_index"}}'
188+
- '{"user": "e", "stars": 3, "date" : "2018-11-29T12:34:12.123456789Z"}'
189+
- '{"index": {"_index": "test_index"}}'
190+
- '{"user": "c", "stars": 4, "date" : "2018-11-29T12:35:12.123456789Z"}'
191+
192+
- do:
193+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
194+
transform.put_transform:
195+
transform_id: "simple-local-remote-transform"
196+
body: >
197+
{
198+
"source": { "index": ["test_index", "my_remote_cluster:test_index"] },
199+
"dest": { "index": "simple-local-remote-transform" },
200+
"pivot": {
201+
"group_by": { "user": {"terms": {"field": "user"}}},
202+
"aggs": {
203+
"avg_stars": {"avg": {"field": "stars"}},
204+
"count": {"value_count": {"field": "user"}}
205+
}
206+
}
207+
}
208+
- match: { acknowledged: true }
209+
210+
- do:
211+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
212+
transform.start_transform:
213+
transform_id: "simple-local-remote-transform"
214+
- match: { acknowledged: true }
215+
216+
- do:
217+
transform.get_transform_stats:
218+
transform_id: "simple-local-remote-transform"
219+
- match: { count: 1 }
220+
- match: { transforms.0.id: "simple-local-remote-transform" }
221+
- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
222+
223+
- do:
224+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
225+
transform.stop_transform:
226+
transform_id: "simple-local-remote-transform"
227+
wait_for_completion: true
228+
wait_for_checkpoint: true
229+
- match: { acknowledged: true }
230+
231+
- do:
232+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
233+
transform.get_transform_stats:
234+
transform_id: "simple-local-remote-transform"
235+
- match: { count: 1 }
236+
- match: { transforms.0.id: "simple-local-remote-transform" }
237+
- match: { transforms.0.state: "stopped" }
238+
- match: { transforms.0.checkpointing.last.checkpoint: 1 }
239+
240+
# workaround: refresh dest index, to be removed, see gh #51154
241+
- do:
242+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
243+
indices.refresh:
244+
index: simple-local-remote-transform
245+
246+
- do:
247+
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
248+
search:
249+
rest_total_hits_as_int: true
250+
index: simple-local-remote-transform
251+
sort: user
252+
253+
- match: { hits.total: 5 }
254+
- match: { hits.hits.0._index: simple-local-remote-transform }
255+
- match: { hits.hits.0._source.avg_stars: 3.5 }
256+
- match: { hits.hits.0._source.count: 6 }
257+
- match: { hits.hits.0._source.user: a }
258+
- match: { hits.hits.1._source.avg_stars: 2.25 }
259+
- match: { hits.hits.1._source.user: b }
260+
- match: { hits.hits.3._source.count: 2 }
261+
- match: { hits.hits.3._source.user: d }

0 commit comments

Comments
 (0)