Skip to content

Commit 5e3ba8a

Browse files
kiawinmartijnvg
kiawin
authored and
martijnvg
committed
Enable convert processor to support Long and Double. (#27957)
Closes #23085
1 parent 6f52fbe commit 5e3ba8a

File tree

3 files changed

+150
-6
lines changed

3 files changed

+150
-6
lines changed

‎docs/reference/ingest/ingest-node.asciidoc‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -778,16 +778,17 @@ Accepts a single value or an array of values.
778778
Converts an existing field's value to a different type, such as converting a string to an integer.
779779
If the field value is an array, all members will be converted.
780780

781-
The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`.
781+
The supported types include: `integer`, `long`, `float`, `double`, `string`, `boolean`, and `auto`.
782782

783783
Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
784784
false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.
785785

786786
Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
787-
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And
788-
a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot
789-
be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In
790-
such a case, `target_field` will still be updated with the unconverted field value.
787+
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. Do note
788+
that float takes precedence of double in `auto`. A value of `"242.15"` will "automatically" be converted to
789+
`242.15` of type `float`. If a provided field cannot be appropriately converted, the Convert Processor will
790+
still process successfully and leave the field value as-is. In such a case, `target_field` will
791+
still be updated with the unconverted field value.
791792

792793
[[convert-options]]
793794
.Convert Options

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,24 @@ public Object convert(Object value) {
4848
}
4949

5050
}
51+
}, LONG {
52+
@Override
53+
public Object convert(Object value) {
54+
try {
55+
return Long.parseLong(value.toString());
56+
} catch(NumberFormatException e) {
57+
throw new IllegalArgumentException("unable to convert [" + value + "] to long", e);
58+
}
59+
}
60+
}, DOUBLE {
61+
@Override
62+
public Object convert(Object value) {
63+
try {
64+
return Double.parseDouble(value.toString());
65+
} catch(NumberFormatException e) {
66+
throw new IllegalArgumentException("unable to convert [" + value + "] to double", e);
67+
}
68+
}
5169
}, FLOAT {
5270
@Override
5371
public Object convert(Object value) {
@@ -81,13 +99,19 @@ public Object convert(Object value) {
8199
}
82100
try {
83101
return BOOLEAN.convert(value);
84-
} catch (IllegalArgumentException e) { }
102+
} catch (IllegalArgumentException e) {}
85103
try {
86104
return INTEGER.convert(value);
87105
} catch (IllegalArgumentException e) {}
106+
try {
107+
return LONG.convert(value);
108+
} catch (IllegalArgumentException e) {}
88109
try {
89110
return FLOAT.convert(value);
90111
} catch (IllegalArgumentException e) {}
112+
try {
113+
return DOUBLE.convert(value);
114+
} catch (IllegalArgumentException e) {}
91115
return value;
92116
}
93117
};

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import static org.hamcrest.Matchers.containsString;
3737
import static org.hamcrest.Matchers.equalTo;
3838
import static org.hamcrest.Matchers.sameInstance;
39+
import static org.hamcrest.Matchers.not;
3940

4041
public class ConvertProcessorTests extends ESTestCase {
4142

@@ -79,6 +80,92 @@ public void testConvertIntError() throws Exception {
7980
}
8081
}
8182

83+
public void testConvertLong() throws Exception {
84+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
85+
Map<String, Long> expectedResult = new HashMap<>();
86+
long randomLong = randomLong();
87+
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomLong);
88+
expectedResult.put(fieldName, randomLong);
89+
90+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
91+
processor.execute(ingestDocument);
92+
assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong));
93+
}
94+
95+
public void testConvertLongList() throws Exception {
96+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
97+
int numItems = randomIntBetween(1, 10);
98+
List<String> fieldValue = new ArrayList<>();
99+
List<Long> expectedList = new ArrayList<>();
100+
for (int j = 0; j < numItems; j++) {
101+
long randomLong = randomLong();
102+
fieldValue.add(Long.toString(randomLong));
103+
expectedList.add(randomLong);
104+
}
105+
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
106+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
107+
processor.execute(ingestDocument);
108+
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
109+
}
110+
111+
public void testConvertLongError() throws Exception {
112+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
113+
String fieldName = RandomDocumentPicks.randomFieldName(random());
114+
String value = "string-" + randomAlphaOfLengthBetween(1, 10);
115+
ingestDocument.setFieldValue(fieldName, value);
116+
117+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
118+
try {
119+
processor.execute(ingestDocument);
120+
fail("processor execute should have failed");
121+
} catch(IllegalArgumentException e) {
122+
assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to long"));
123+
}
124+
}
125+
126+
public void testConvertDouble() throws Exception {
127+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
128+
Map<String, Double> expectedResult = new HashMap<>();
129+
double randomDouble = randomDouble();
130+
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomDouble);
131+
expectedResult.put(fieldName, randomDouble);
132+
133+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
134+
processor.execute(ingestDocument);
135+
assertThat(ingestDocument.getFieldValue(fieldName, Double.class), equalTo(randomDouble));
136+
}
137+
138+
public void testConvertDoubleList() throws Exception {
139+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
140+
int numItems = randomIntBetween(1, 10);
141+
List<String> fieldValue = new ArrayList<>();
142+
List<Double> expectedList = new ArrayList<>();
143+
for (int j = 0; j < numItems; j++) {
144+
double randomDouble = randomDouble();
145+
fieldValue.add(Double.toString(randomDouble));
146+
expectedList.add(randomDouble);
147+
}
148+
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
149+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
150+
processor.execute(ingestDocument);
151+
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
152+
}
153+
154+
public void testConvertDoubleError() throws Exception {
155+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
156+
String fieldName = RandomDocumentPicks.randomFieldName(random());
157+
String value = "string-" + randomAlphaOfLengthBetween(1, 10);
158+
ingestDocument.setFieldValue(fieldName, value);
159+
160+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
161+
try {
162+
processor.execute(ingestDocument);
163+
fail("processor execute should have failed");
164+
} catch(IllegalArgumentException e) {
165+
assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to double"));
166+
}
167+
}
168+
82169
public void testConvertFloat() throws Exception {
83170
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
84171
Map<String, Float> expectedResult = new HashMap<>();
@@ -231,6 +318,16 @@ public void testConvertStringList() throws Exception {
231318
randomValue = randomBoolean;
232319
randomValueString = Boolean.toString(randomBoolean);
233320
break;
321+
case 3:
322+
long randomLong = randomLong();
323+
randomValue = randomLong;
324+
randomValueString = Long.toString(randomLong);
325+
break;
326+
case 4:
327+
double randomDouble = randomDouble();
328+
randomValue = randomDouble;
329+
randomValueString = Double.toString(randomDouble);
330+
break;
234331
default:
235332
throw new UnsupportedOperationException();
236333
}
@@ -342,6 +439,28 @@ public void testAutoConvertMatchInteger() throws Exception {
342439
assertThat(convertedValue, equalTo(randomInt));
343440
}
344441

442+
public void testAutoConvertMatchLong() throws Exception {
443+
long randomLong = randomLong();
444+
String randomString = Long.toString(randomLong);
445+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
446+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
447+
processor.execute(ingestDocument);
448+
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
449+
assertThat(convertedValue, equalTo(randomLong));
450+
}
451+
452+
public void testAutoConvertDoubleNotMatched() throws Exception {
453+
double randomDouble = randomDouble();
454+
String randomString = Double.toString(randomDouble);
455+
float randomFloat = Float.parseFloat(randomString);
456+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
457+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
458+
processor.execute(ingestDocument);
459+
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
460+
assertThat(convertedValue, not(randomDouble));
461+
assertThat(convertedValue, equalTo(randomFloat));
462+
}
463+
345464
public void testAutoConvertMatchFloat() throws Exception {
346465
float randomFloat = randomFloat();
347466
String randomString = Float.toString(randomFloat);

0 commit comments

Comments
 (0)