Skip to content

Commit f74d2c8

Browse files
author
Hendrik Muhs
committed
fix more tests
1 parent 05322a8 commit f74d2c8

File tree

2 files changed

+107
-69
lines changed

2 files changed

+107
-69
lines changed

‎x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java‎

Lines changed: 68 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,9 @@
2929
public class TransformGetAndGetStatsIT extends TransformRestTestCase {
3030

3131
private static final String TEST_USER_NAME = "transform_user";
32-
private static final String BASIC_AUTH_VALUE_TRANSFORM_USER =
33-
basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
32+
private static final String BASIC_AUTH_VALUE_TRANSFORM_USER = basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
3433
private static final String TEST_ADMIN_USER_NAME = "transform_admin";
35-
private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN =
36-
basicAuthHeaderValue(TEST_ADMIN_USER_NAME, TEST_PASSWORD_SECURE_STRING);
34+
private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN = basicAuthHeaderValue(TEST_ADMIN_USER_NAME, TEST_PASSWORD_SECURE_STRING);
3735

3836
private static boolean indicesCreated = false;
3937

@@ -101,13 +99,13 @@ public void testGetAndGetStats() throws Exception {
10199
stats = entityAsMap(client().performRequest(getRequest));
102100
assertEquals(3, XContentMapValues.extractValue("count", stats));
103101

104-
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
102+
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", stats);
105103
// Verify that both transforms have valid stats
106104
for (Map<String, Object> transformStats : transformsStats) {
107-
Map<String, Object> stat = (Map<String, Object>)transformStats.get("stats");
108-
assertThat("documents_processed is not > 0.", ((Integer)stat.get("documents_processed")), greaterThan(0));
109-
assertThat("search_total is not > 0.", ((Integer)stat.get("search_total")), greaterThan(0));
110-
assertThat("pages_processed is not > 0.", ((Integer)stat.get("pages_processed")), greaterThan(0));
105+
Map<String, Object> stat = (Map<String, Object>) transformStats.get("stats");
106+
assertThat("documents_processed is not > 0.", ((Integer) stat.get("documents_processed")), greaterThan(0));
107+
assertThat("search_total is not > 0.", ((Integer) stat.get("search_total")), greaterThan(0));
108+
assertThat("pages_processed is not > 0.", ((Integer) stat.get("pages_processed")), greaterThan(0));
111109
/* TODO progress is now checkpoint progress and it may be that no checkpoint is in progress here
112110
Map<String, Object> progress =
113111
(Map<String, Object>)XContentMapValues.extractValue("checkpointing.next.checkpoint_progress", transformStats);
@@ -122,7 +120,7 @@ public void testGetAndGetStats() throws Exception {
122120
stats = entityAsMap(client().performRequest(getRequest));
123121
assertEquals(1, XContentMapValues.extractValue("count", stats));
124122

125-
transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
123+
transformsStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", stats);
126124
assertEquals(1, transformsStats.size());
127125
assertEquals("stopped", XContentMapValues.extractValue("state", transformsStats.get(0)));
128126
assertNull(XContentMapValues.extractValue("checkpointing.next.position", transformsStats.get(0)));
@@ -133,12 +131,11 @@ public void testGetAndGetStats() throws Exception {
133131
stats = entityAsMap(client().performRequest(getRequest));
134132
assertEquals(1, XContentMapValues.extractValue("count", stats));
135133

136-
transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
134+
transformsStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", stats);
137135
assertEquals(1, transformsStats.size());
138136
assertThat(XContentMapValues.extractValue("state", transformsStats.get(0)), oneOf("started", "indexing"));
139137
assertEquals(1, XContentMapValues.extractValue("checkpointing.last.checkpoint", transformsStats.get(0)));
140138

141-
142139
// check all the different ways to retrieve all transforms
143140
getRequest = createRequestWithAuth("GET", getTransformEndpoint(), authHeader);
144141
Map<String, Object> transforms = entityAsMap(client().performRequest(getRequest));
@@ -165,26 +162,27 @@ public void testGetPersistedStatsWithoutTask() throws Exception {
165162
stopTransform("pivot_stats_1", false);
166163

167164
// Get rid of the first transform task, but keep the configuration
168-
client().performRequest(new Request("POST", "_tasks/_cancel?actions="+TransformField.TASK_NAME+"*"));
165+
client().performRequest(new Request("POST", "_tasks/_cancel?actions=" + TransformField.TASK_NAME + "*"));
169166

170167
// Verify that the task is gone
171-
Map<String, Object> tasks =
172-
entityAsMap(client().performRequest(new Request("GET", "_tasks?actions="+TransformField.TASK_NAME+"*")));
173-
assertTrue(((Map<?, ?>)XContentMapValues.extractValue("nodes", tasks)).isEmpty());
168+
Map<String, Object> tasks = entityAsMap(
169+
client().performRequest(new Request("GET", "_tasks?actions=" + TransformField.TASK_NAME + "*"))
170+
);
171+
assertTrue(((Map<?, ?>) XContentMapValues.extractValue("nodes", tasks)).isEmpty());
174172

175173
createPivotReviewsTransform("pivot_stats_2", "pivot_reviews_stats_2", null);
176174
startAndWaitForTransform("pivot_stats_2", "pivot_reviews_stats_2");
177175

178176
Request getRequest = createRequestWithAuth("GET", getTransformEndpoint() + "_stats", BASIC_AUTH_VALUE_TRANSFORM_ADMIN);
179177
Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
180178
assertEquals(2, XContentMapValues.extractValue("count", stats));
181-
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
179+
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", stats);
182180
// Verify that both transforms, the one with the task and the one without have statistics
183181
for (Map<String, Object> transformStats : transformsStats) {
184-
Map<String, Object> stat = (Map<String, Object>)transformStats.get("stats");
185-
assertThat(((Integer)stat.get("documents_processed")), greaterThan(0));
186-
assertThat(((Integer)stat.get("search_total")), greaterThan(0));
187-
assertThat(((Integer)stat.get("pages_processed")), greaterThan(0));
182+
Map<String, Object> stat = (Map<String, Object>) transformStats.get("stats");
183+
assertThat(((Integer) stat.get("documents_processed")), greaterThan(0));
184+
assertThat(((Integer) stat.get("search_total")), greaterThan(0));
185+
assertThat(((Integer) stat.get("pages_processed")), greaterThan(0));
188186
}
189187
}
190188

@@ -202,13 +200,13 @@ public void testGetProgressStatsWithPivotQuery() throws Exception {
202200
Request getRequest = createRequestWithAuth("GET", getTransformEndpoint() + transformId + "/_stats", authHeader);
203201
Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
204202
assertEquals(1, XContentMapValues.extractValue("count", stats));
205-
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
203+
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", stats);
206204
// Verify that the transform has stats and the total docs process matches the expected
207205
for (Map<String, Object> transformStats : transformsStats) {
208-
Map<String, Object> stat = (Map<String, Object>)transformStats.get("stats");
209-
assertThat("documents_processed is not > 0.", ((Integer)stat.get("documents_processed")), greaterThan(0));
210-
assertThat("search_total is not > 0.", ((Integer)stat.get("search_total")), greaterThan(0));
211-
assertThat("pages_processed is not > 0.", ((Integer)stat.get("pages_processed")), greaterThan(0));
206+
Map<String, Object> stat = (Map<String, Object>) transformStats.get("stats");
207+
assertThat("documents_processed is not > 0.", ((Integer) stat.get("documents_processed")), greaterThan(0));
208+
assertThat("search_total is not > 0.", ((Integer) stat.get("search_total")), greaterThan(0));
209+
assertThat("pages_processed is not > 0.", ((Integer) stat.get("pages_processed")), greaterThan(0));
212210
/* TODO progress is now checkpoint progress and it may be that no checkpoint is in progress here
213211
Map<String, Object> progress =
214212
(Map<String, Object>)XContentMapValues.extractValue("checkpointing.next.checkpoint_progress", transformStats);
@@ -226,8 +224,12 @@ public void testGetStatsWithContinuous() throws Exception {
226224
String transformSrc = "reviews_cont_pivot_test";
227225
createReviewsIndex(transformSrc);
228226
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, null);
229-
String config = "{ \"dest\": {\"index\":\"" + transformDest + "\"},"
230-
+ " \"source\": {\"index\":\"" + transformSrc + "\"},"
227+
String config = "{ \"dest\": {\"index\":\""
228+
+ transformDest
229+
+ "\"},"
230+
+ " \"source\": {\"index\":\""
231+
+ transformSrc
232+
+ "\"},"
231233
+ " \"frequency\": \"1s\","
232234
+ " \"sync\": {\"time\":{\"field\": \"timestamp\", \"delay\": \"1s\"}},"
233235
+ " \"pivot\": {"
@@ -251,20 +253,28 @@ public void testGetStatsWithContinuous() throws Exception {
251253

252254
Request getRequest = createRequestWithAuth("GET", getTransformEndpoint() + transformId + "/_stats", null);
253255
Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
254-
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
256+
List<Map<String, Object>> transformsStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", stats);
255257
assertEquals(1, transformsStats.size());
256-
// No continuous checkpoints have been seen and thus all exponential averages should be 0.0
258+
// No continuous checkpoints have been seen and thus all exponential averages should be equal to the batch stats
257259
for (Map<String, Object> transformStats : transformsStats) {
258-
transformStats = (Map<String, Object>)transformStats.get("stats");
259-
assertThat("exponential_avg_checkpoint_duration_ms is not 0.0",
260-
transformStats.get("exponential_avg_checkpoint_duration_ms"),
261-
equalTo(0.0));
262-
assertThat("exponential_avg_documents_indexed is not 0.0",
263-
transformStats.get("exponential_avg_documents_indexed"),
264-
equalTo(0.0));
265-
assertThat("exponential_avg_documents_processed is not 0.0",
260+
transformStats = (Map<String, Object>) transformStats.get("stats");
261+
assertThat(transformStats.get("documents_processed"), equalTo(1000));
262+
assertThat(transformStats.get("documents_indexed"), equalTo(27));
263+
assertThat(
264+
"exponential_avg_checkpoint_duration_ms is not 0.0",
265+
(Double) transformStats.get("exponential_avg_checkpoint_duration_ms"),
266+
greaterThan(0.0)
267+
);
268+
assertThat(
269+
"exponential_avg_documents_indexed does not match documents_indexed",
270+
(Double) transformStats.get("exponential_avg_documents_indexed"),
271+
equalTo(((Integer) transformStats.get("documents_indexed")).doubleValue())
272+
);
273+
assertThat(
274+
"exponential_avg_documents_processed does not match documents_processed",
266275
transformStats.get("exponential_avg_documents_processed"),
267-
equalTo(0.0));
276+
equalTo(((Integer) transformStats.get("documents_processed")).doubleValue())
277+
);
268278
}
269279

270280
int numDocs = 10;
@@ -296,23 +306,27 @@ public void testGetStatsWithContinuous() throws Exception {
296306
// We should now have exp avgs since we have processed a continuous checkpoint
297307
assertBusy(() -> {
298308
Map<String, Object> statsResponse = entityAsMap(client().performRequest(getRequest));
299-
List<Map<String, Object>> contStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", statsResponse);
309+
List<Map<String, Object>> contStats = (List<Map<String, Object>>) XContentMapValues.extractValue("transforms", statsResponse);
300310
assertEquals(1, contStats.size());
301311
for (Map<String, Object> transformStats : contStats) {
302-
Map<String, Object> statsObj = (Map<String, Object>)transformStats.get("stats");
303-
assertThat("exponential_avg_checkpoint_duration_ms is 0",
304-
(Double)statsObj.get("exponential_avg_checkpoint_duration_ms"),
305-
greaterThan(0.0));
306-
assertThat("exponential_avg_documents_indexed is 0",
307-
(Double)statsObj.get("exponential_avg_documents_indexed"),
308-
greaterThan(0.0));
309-
assertThat("exponential_avg_documents_processed is 0",
310-
(Double)statsObj.get("exponential_avg_documents_processed"),
311-
greaterThan(0.0));
312-
Map<String, Object> checkpointing = (Map<String, Object>)transformStats.get("checkpointing");
313-
assertThat("changes_last_detected_at is null",
314-
checkpointing.get("changes_last_detected_at"),
315-
is(notNullValue()));
312+
Map<String, Object> statsObj = (Map<String, Object>) transformStats.get("stats");
313+
assertThat(
314+
"exponential_avg_checkpoint_duration_ms is 0",
315+
(Double) statsObj.get("exponential_avg_checkpoint_duration_ms"),
316+
greaterThan(0.0)
317+
);
318+
assertThat(
319+
"exponential_avg_documents_indexed is 0",
320+
(Double) statsObj.get("exponential_avg_documents_indexed"),
321+
greaterThan(0.0)
322+
);
323+
assertThat(
324+
"exponential_avg_documents_processed is 0",
325+
(Double) statsObj.get("exponential_avg_documents_processed"),
326+
greaterThan(0.0)
327+
);
328+
Map<String, Object> checkpointing = (Map<String, Object>) transformStats.get("checkpointing");
329+
assertThat("changes_last_detected_at is null", checkpointing.get("changes_last_detected_at"), is(notNullValue()));
316330
}
317331
}, 120, TimeUnit.SECONDS);
318332
}

‎x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,14 @@ public void testUsage() throws Exception {
5252
startAndWaitForTransform("test_usage", "pivot_reviews");
5353
stopTransform("test_usage", false);
5454

55-
Request statsExistsRequest = new Request("GET",
56-
TransformInternalIndexConstants.LATEST_INDEX_NAME+"/_search?q=" +
57-
INDEX_DOC_TYPE.getPreferredName() + ":" +
58-
TransformStoredDoc.NAME);
55+
Request statsExistsRequest = new Request(
56+
"GET",
57+
TransformInternalIndexConstants.LATEST_INDEX_NAME
58+
+ "/_search?q="
59+
+ INDEX_DOC_TYPE.getPreferredName()
60+
+ ":"
61+
+ TransformStoredDoc.NAME
62+
);
5963
// Verify that we have one stat document
6064
assertBusy(() -> {
6165
Map<String, Object> hasStatsMap = entityAsMap(client().performRequest(statsExistsRequest));
@@ -66,13 +70,21 @@ public void testUsage() throws Exception {
6670

6771
Request getRequest = new Request("GET", getTransformEndpoint() + "test_usage/_stats");
6872
Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
69-
Map<String, Integer> expectedStats = new HashMap<>();
70-
for(String statName : PROVIDED_STATS) {
73+
Map<String, Double> expectedStats = new HashMap<>();
74+
for (String statName : PROVIDED_STATS) {
7175
@SuppressWarnings("unchecked")
72-
List<Integer> specificStatistic = ((List<Integer>)XContentMapValues.extractValue("transforms.stats." + statName, stats));
76+
List<Object> specificStatistic = (List<Object>) (XContentMapValues.extractValue("transforms.stats." + statName, stats));
7377
assertNotNull(specificStatistic);
74-
Integer statistic = (specificStatistic).get(0);
75-
expectedStats.put(statName, statistic);
78+
expectedStats.put(statName, extractStatsAsDouble(specificStatistic.get(0)));
79+
}
80+
81+
getRequest = new Request("GET", getTransformEndpoint() + "test_usage_continuous/_stats");
82+
stats = entityAsMap(client().performRequest(getRequest));
83+
for (String statName : PROVIDED_STATS) {
84+
@SuppressWarnings("unchecked")
85+
List<Object> specificStatistic = (List<Object>) (XContentMapValues.extractValue("transforms.stats." + statName, stats));
86+
assertNotNull(specificStatistic);
87+
expectedStats.compute(statName, (key, value) -> value + extractStatsAsDouble(specificStatistic.get(0)));
7688
}
7789

7890
// Simply because we wait for continuous to reach checkpoint 1, does not mean that the statistics are written yet.
@@ -85,20 +97,22 @@ public void testUsage() throws Exception {
8597
assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", statsMap));
8698
assertEquals(2, XContentMapValues.extractValue("transform.transforms.stopped", statsMap));
8799
assertEquals(1, XContentMapValues.extractValue("transform.transforms.started", statsMap));
88-
for(String statName : PROVIDED_STATS) {
100+
for (String statName : PROVIDED_STATS) {
89101
if (statName.equals(TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName())
90-
||statName.equals(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName())) {
102+
|| statName.equals(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName())) {
91103
continue;
92104
}
93-
assertEquals("Incorrect stat " + statName,
94-
expectedStats.get(statName) * 2,
95-
XContentMapValues.extractValue("transform.stats." + statName, statsMap));
105+
assertEquals(
106+
"Incorrect stat " + statName,
107+
expectedStats.get(statName).doubleValue(),
108+
extractStatsAsDouble(XContentMapValues.extractValue("transform.stats." + statName, statsMap)),
109+
0.0001
110+
);
96111
}
97112
// Refresh the index so that statistics are searchable
98113
refreshIndex(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME);
99114
}, 60, TimeUnit.SECONDS);
100115

101-
102116
stopTransform("test_usage_continuous", false);
103117

104118
usageResponse = client().performRequest(new Request("GET", "_xpack/usage"));
@@ -107,4 +121,14 @@ public void testUsage() throws Exception {
107121
assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", usageAsMap));
108122
assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap));
109123
}
124+
125+
private double extractStatsAsDouble(Object statsObject) {
126+
if (statsObject instanceof Integer) {
127+
return ((Integer) statsObject).doubleValue();
128+
} else if (statsObject instanceof Double) {
129+
return (Double) statsObject;
130+
}
131+
fail("unexpected value type for stats");
132+
return 0;
133+
}
110134
}

0 commit comments

Comments
 (0)