Skip to content

Commit 995bed2

Browse files
author
Hendrik Muhs
authored
[Transform] fix transform failure case for percentiles and spa… (#54202)
index null if percentiles could not be calculated due to sparse data fixes #54201
1 parent 6d6227a commit 995bed2

File tree

3 files changed

+123
-2
lines changed

3 files changed

+123
-2
lines changed

‎x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,19 @@
66

77
package org.elasticsearch.xpack.transform.integration;
88

9+
import org.apache.http.entity.ContentType;
10+
import org.apache.http.entity.StringEntity;
911
import org.elasticsearch.client.Request;
12+
import org.elasticsearch.common.Strings;
13+
import org.elasticsearch.common.xcontent.XContentBuilder;
1014
import org.elasticsearch.common.xcontent.support.XContentMapValues;
1115
import org.junit.Before;
1216

1317
import java.io.IOException;
1418
import java.util.List;
1519
import java.util.Map;
1620

21+
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
1722
import static org.hamcrest.Matchers.equalTo;
1823

1924
public class TransformPivotRestSpecialCasesIT extends TransformRestTestCase {
@@ -102,4 +107,110 @@ public void testIndexTemplateMappingClash() throws Exception {
102107
Number actual = (Number) ((List<?>) XContentMapValues.extractValue("hits.hits._source.rating.avg", searchResult)).get(0);
103108
assertEquals(3.878048780, actual.doubleValue(), 0.000001);
104109
}
110+
111+
public void testSparseDataPercentiles() throws Exception {
112+
String indexName = "cpu-utilization";
113+
String transformIndex = "pivot-cpu";
114+
String transformId = "pivot-cpu";
115+
116+
try (XContentBuilder builder = jsonBuilder()) {
117+
builder.startObject();
118+
{
119+
builder.startObject("mappings")
120+
.startObject("properties")
121+
.startObject("host")
122+
.field("type", "keyword")
123+
.endObject()
124+
.startObject("cpu")
125+
.field("type", "integer")
126+
.endObject()
127+
.endObject()
128+
.endObject();
129+
}
130+
builder.endObject();
131+
final StringEntity entity = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
132+
Request req = new Request("PUT", indexName);
133+
req.setEntity(entity);
134+
client().performRequest(req);
135+
}
136+
137+
final StringBuilder bulk = new StringBuilder();
138+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
139+
bulk.append("{\"host\":\"host-1\",\"cpu\": 22}\n");
140+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
141+
bulk.append("{\"host\":\"host-1\",\"cpu\": 55}\n");
142+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
143+
bulk.append("{\"host\":\"host-1\",\"cpu\": 23}\n");
144+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
145+
bulk.append("{\"host\":\"host-2\",\"cpu\": 0}\n");
146+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
147+
bulk.append("{\"host\":\"host-2\",\"cpu\": 99}\n");
148+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
149+
bulk.append("{\"host\":\"host-1\",\"cpu\": 28}\n");
150+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
151+
bulk.append("{\"host\":\"host-1\",\"cpu\": 77}\n");
152+
153+
// missing value for cpu
154+
bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
155+
bulk.append("{\"host\":\"host-3\"}\n");
156+
bulk.append("\r\n");
157+
final Request bulkRequest = new Request("POST", "/_bulk");
158+
bulkRequest.addParameter("refresh", "true");
159+
bulkRequest.setJsonEntity(bulk.toString());
160+
client().performRequest(bulkRequest);
161+
162+
final Request createTransformRequest = new Request("PUT", getTransformEndpoint() + transformId);
163+
164+
String config = "{" + " \"source\": {\"index\":\"" + indexName + "\"}," + " \"dest\": {\"index\":\"" + transformIndex + "\"},";
165+
166+
config += " \"pivot\": {"
167+
+ " \"group_by\": {"
168+
+ " \"host\": {"
169+
+ " \"terms\": {"
170+
+ " \"field\": \"host\""
171+
+ " } } },"
172+
+ " \"aggregations\": {"
173+
+ " \"p\": {"
174+
+ " \"percentiles\": {"
175+
+ " \"field\": \"cpu\""
176+
+ " } }"
177+
+ " } }"
178+
+ "}";
179+
180+
createTransformRequest.setJsonEntity(config);
181+
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
182+
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
183+
184+
startAndWaitForTransform(transformId, transformIndex);
185+
assertTrue(indexExists(transformIndex));
186+
187+
Map<String, Object> indexStats = getAsMap(transformIndex + "/_stats");
188+
assertEquals(3, XContentMapValues.extractValue("_all.total.docs.count", indexStats));
189+
190+
// get and check some data
191+
Map<String, Object> searchResult = getAsMap(transformIndex + "/_search?q=host:host-1");
192+
193+
assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
194+
@SuppressWarnings("unchecked")
195+
Map<String, Object> percentiles = (Map<String, Object>) ((List<?>) XContentMapValues.extractValue(
196+
"hits.hits._source.p",
197+
searchResult
198+
)).get(0);
199+
200+
assertEquals(28.0, (double) percentiles.get("50"), 0.000001);
201+
assertEquals(77.0, (double) percentiles.get("99"), 0.000001);
202+
203+
searchResult = getAsMap(transformIndex + "/_search?q=host:host-3");
204+
assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
205+
206+
@SuppressWarnings("unchecked")
207+
Map<String, Object> percentilesEmpty = (Map<String, Object>) ((List<?>) XContentMapValues.extractValue(
208+
"hits.hits._source.p",
209+
searchResult
210+
)).get(0);
211+
assertTrue(percentilesEmpty.containsKey("50"));
212+
assertNull(percentilesEmpty.get("50"));
213+
assertTrue(percentilesEmpty.containsKey("99"));
214+
assertNull(percentilesEmpty.get("99"));
215+
}
105216
}

‎x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,16 @@ static class PercentilesAggExtractor implements AggValueExtractor {
205205
@Override
206206
public Object value(Aggregation agg, Map<String, String> fieldTypeMap, String lookupFieldPrefix) {
207207
Percentiles aggregation = (Percentiles) agg;
208-
209208
HashMap<String, Double> percentiles = new HashMap<>();
210209

211210
for (Percentile p : aggregation) {
212-
percentiles.put(OutputFieldNameConverter.fromDouble(p.getPercent()), p.getValue());
211+
// in case of sparse data percentiles might not have data, in this case it returns NaN,
212+
// we need to guard the output and set null in this case
213+
if (Numbers.isValidDouble(p.getValue()) == false) {
214+
percentiles.put(OutputFieldNameConverter.fromDouble(p.getPercent()), null);
215+
} else {
216+
percentiles.put(OutputFieldNameConverter.fromDouble(p.getPercent()), p.getValue());
217+
}
213218
}
214219

215220
return percentiles;

‎x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -798,6 +798,11 @@ public void testPercentilesAggExtractor() {
798798
);
799799
}
800800

801+
public void testPercentilesAggExtractorNaN() {
802+
Aggregation agg = createPercentilesAgg("p_agg", Arrays.asList(new Percentile(1, Double.NaN), new Percentile(50, Double.NaN)));
803+
assertThat(AggregationResultUtils.getExtractor(agg).value(agg, Collections.emptyMap(), ""), equalTo(asMap("1", null, "50", null)));
804+
}
805+
801806
public static SingleBucketAggregation createSingleBucketAgg(String name, long docCount, Aggregation... subAggregations) {
802807
SingleBucketAggregation agg = mock(SingleBucketAggregation.class);
803808
when(agg.getDocCount()).thenReturn(docCount);

0 commit comments

Comments
 (0)