Skip to content

Commit 5ed38f0

Browse files
bharath-techiebharath-techie
committed
Changes to support IP
Signed-off-by: bharath-techie <[email protected]>
1 parent e9f77e3 commit 5ed38f0

File tree

12 files changed

+165
-24
lines changed

12 files changed

+165
-24
lines changed

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+11-7
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase {
5656
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
5757
.build();
5858

59-
private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean ipdim) {
59+
private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean wildcard) {
6060
try {
6161
return jsonBuilder().startObject()
6262
.startObject("composite")
@@ -68,7 +68,7 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
6868
.endObject()
6969
.startArray("ordered_dimensions")
7070
.startObject()
71-
.field("name", getDim(invalidDim, ipdim))
71+
.field("name", getDim(invalidDim, wildcard))
7272
.endObject()
7373
.startObject()
7474
.field("name", "keyword_dv")
@@ -104,6 +104,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
104104
.endObject()
105105
.startObject("ip")
106106
.field("type", "ip")
107+
.field("doc_values", true)
108+
.endObject()
109+
.startObject("wildcard")
110+
.field("type", "wildcard")
107111
.field("doc_values", false)
108112
.endObject()
109113
.endObject()
@@ -362,11 +366,11 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo
362366
return mapping;
363367
}
364368

365-
private static String getDim(boolean hasDocValues, boolean isKeyword) {
369+
private static String getDim(boolean hasDocValues, boolean isWildCard) {
366370
if (hasDocValues) {
367-
return random().nextBoolean() ? "numeric" : "keyword";
368-
} else if (isKeyword) {
369-
return "ip";
371+
return random().nextBoolean() ? "numeric" : random().nextBoolean() ? "keyword" : "ip";
372+
} else if (isWildCard) {
373+
return "wildcard";
370374
}
371375
return "numeric_dv";
372376
}
@@ -748,7 +752,7 @@ public void testUnsupportedDim() {
748752
() -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get()
749753
);
750754
assertEquals(
751-
"Failed to parse mapping [_doc]: unsupported field type associated with dimension [ip] as part of star tree field [startree-1]",
755+
"Failed to parse mapping [_doc]: unsupported field type associated with dimension [wildcard] as part of star tree field [startree-1]",
752756
ex.getMessage()
753757
);
754758
}

‎server/src/main/java/org/opensearch/common/util/FeatureFlags.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public class FeatureFlags {
100100
* aggregations.
101101
*/
102102
public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled";
103-
public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
103+
public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, true, Property.NodeScope);
104104

105105
/**
106106
* Gates the functionality of application based configuration templates.

‎server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
3636
import org.opensearch.index.mapper.CompositeMappedFieldType;
3737
import org.opensearch.index.mapper.DocCountFieldMapper;
38+
import org.opensearch.index.mapper.IpFieldMapper;
3839
import org.opensearch.index.mapper.KeywordFieldMapper;
3940
import org.opensearch.index.mapper.MapperService;
4041

@@ -277,7 +278,8 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
277278
}
278279

279280
private boolean isSortedSetField(String field) {
280-
return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType;
281+
return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType
282+
|| mapperService.fieldType(field) instanceof IpFieldMapper.IpFieldType;
281283
}
282284

283285
@Override

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.stream.Collectors;
2525

2626
import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS;
27+
import static org.opensearch.index.compositeindex.datacube.IpDimension.IP;
2728
import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD;
2829

2930
/**
@@ -46,6 +47,8 @@ public static Dimension parseAndCreateDimension(
4647
return new NumericDimension(name);
4748
case KEYWORD:
4849
return new KeywordDimension(name);
50+
case IP:
51+
return new IpDimension(name);
4952
default:
5053
throw new IllegalArgumentException(
5154
String.format(Locale.ROOT, "unsupported field type associated with dimension [%s] as part of star tree field", name)
@@ -71,6 +74,8 @@ public static Dimension parseAndCreateDimension(
7174
return new NumericDimension(name);
7275
case KEYWORD:
7376
return new KeywordDimension(name);
77+
case IP:
78+
return new IpDimension(name);
7479
default:
7580
throw new IllegalArgumentException(
7681
String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name)

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,11 @@ public enum DimensionType {
3333
* Represents a keyword dimension type.
3434
* This is used for dimensions that contain keyword ordinals.
3535
*/
36-
KEYWORD
36+
KEYWORD,
37+
38+
/**
39+
* Represents an IP dimension type.
40+
* This is used for dimensions that contain IP ordinals.
41+
*/
42+
IP
3743
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.compositeindex.datacube;
10+
11+
import org.apache.lucene.index.DocValuesType;
12+
import org.opensearch.common.annotation.ExperimentalApi;
13+
import org.opensearch.core.xcontent.XContentBuilder;
14+
import org.opensearch.index.mapper.CompositeDataCubeFieldType;
15+
16+
import java.io.IOException;
17+
import java.util.List;
18+
import java.util.Objects;
19+
import java.util.function.Consumer;
20+
21+
/**
22+
* Composite index keyword dimension class
23+
*
24+
* @opensearch.experimental
25+
*/
26+
@ExperimentalApi
27+
public class IpDimension implements Dimension {
28+
public static final String IP = "ip";
29+
private final String field;
30+
31+
public IpDimension(String field) {
32+
this.field = field;
33+
}
34+
35+
@Override
36+
public String getField() {
37+
return field;
38+
}
39+
40+
@Override
41+
public int getNumSubDimensions() {
42+
return 1;
43+
}
44+
45+
@Override
46+
public void setDimensionValues(Long value, Consumer<Long> dimSetter) {
47+
// This will set the keyword dimension value's ordinal
48+
dimSetter.accept(value);
49+
}
50+
51+
@Override
52+
public List<String> getSubDimensionNames() {
53+
return List.of(field);
54+
}
55+
56+
@Override
57+
public DocValuesType getDocValuesType() {
58+
return DocValuesType.SORTED_SET;
59+
}
60+
61+
@Override
62+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
63+
builder.startObject();
64+
builder.field(CompositeDataCubeFieldType.NAME, field);
65+
builder.field(CompositeDataCubeFieldType.TYPE, IP);
66+
builder.endObject();
67+
return builder;
68+
}
69+
70+
@Override
71+
public boolean equals(Object o) {
72+
if (this == o) return true;
73+
if (o == null || getClass() != o.getClass()) return false;
74+
IpDimension dimension = (IpDimension) o;
75+
return Objects.equals(field, dimension.getField());
76+
}
77+
78+
@Override
79+
public int hashCode() {
80+
return Objects.hash(field);
81+
}
82+
}

‎server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java

+7
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.opensearch.common.collect.Tuple;
4848
import org.opensearch.common.logging.DeprecationLogger;
4949
import org.opensearch.common.network.InetAddresses;
50+
import org.opensearch.index.compositeindex.datacube.DimensionType;
5051
import org.opensearch.index.fielddata.IndexFieldData;
5152
import org.opensearch.index.fielddata.ScriptDocValues;
5253
import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
@@ -62,6 +63,7 @@
6263
import java.util.Collections;
6364
import java.util.List;
6465
import java.util.Map;
66+
import java.util.Optional;
6567
import java.util.function.BiFunction;
6668
import java.util.function.Supplier;
6769

@@ -155,6 +157,11 @@ public IpFieldMapper build(BuilderContext context) {
155157
);
156158
}
157159

160+
@Override
161+
public Optional<DimensionType> getSupportedDataCubeDimensionType() {
162+
return Optional.of(DimensionType.IP);
163+
}
164+
158165
}
159166

160167
public static final TypeParser PARSER = new TypeParser((n, c) -> {

‎server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeKeywordDocValuesFormatTests.java

+32-5
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import org.apache.lucene.document.Document;
1212
import org.apache.lucene.document.Field;
13+
import org.apache.lucene.document.InetAddressPoint;
1314
import org.apache.lucene.document.NumericDocValuesField;
1415
import org.apache.lucene.document.SortedNumericDocValuesField;
1516
import org.apache.lucene.document.SortedSetDocValuesField;
@@ -25,6 +26,7 @@
2526
import org.apache.lucene.tests.util.TestUtil;
2627
import org.apache.lucene.util.BytesRef;
2728
import org.opensearch.common.lucene.Lucene;
29+
import org.opensearch.common.network.InetAddresses;
2830
import org.opensearch.core.xcontent.XContentBuilder;
2931
import org.opensearch.index.codec.composite.CompositeIndexFieldInfo;
3032
import org.opensearch.index.codec.composite.CompositeIndexReader;
@@ -36,6 +38,8 @@
3638
import org.opensearch.index.mapper.NumberFieldMapper;
3739

3840
import java.io.IOException;
41+
import java.net.InetAddress;
42+
import java.util.Arrays;
3943
import java.util.HashMap;
4044
import java.util.HashSet;
4145
import java.util.List;
@@ -65,12 +69,15 @@ public void testStarTreeKeywordDocValues() throws IOException {
6569
doc.add(new SortedNumericDocValuesField("sndv", 1));
6670
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
6771
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
72+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
6873
iw.addDocument(doc);
6974
doc = new Document();
7075
doc.add(new StringField("_id", "2", Field.Store.NO));
7176
doc.add(new SortedNumericDocValuesField("sndv", 1));
7277
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
7378
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
79+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
80+
7481
iw.addDocument(doc);
7582
iw.flush();
7683
iw.deleteDocuments(new Term("_id", "2"));
@@ -80,12 +87,14 @@ public void testStarTreeKeywordDocValues() throws IOException {
8087
doc.add(new SortedNumericDocValuesField("sndv", 2));
8188
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
8289
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
90+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
8391
iw.addDocument(doc);
8492
doc = new Document();
8593
doc.add(new StringField("_id", "4", Field.Store.NO));
8694
doc.add(new SortedNumericDocValuesField("sndv", 2));
8795
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
8896
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
97+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
8998
iw.addDocument(doc);
9099
iw.flush();
91100
iw.deleteDocuments(new Term("_id", "4"));
@@ -166,6 +175,9 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {
166175

167176
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef(keyword2Value)));
168177
map.put(keyword1Value + "-" + keyword2Value, sndvValue + map.getOrDefault(keyword1Value + "-" + keyword2Value, 0));
178+
doc.add(
179+
new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10." + i))))
180+
);
169181
iw.addDocument(doc);
170182
documents.put(id, doc);
171183
}
@@ -221,9 +233,7 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {
221233
SortedSetStarTreeValuesIterator k1 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator(
222234
"keyword1"
223235
);
224-
SortedSetStarTreeValuesIterator k2 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator(
225-
"keyword2"
226-
);
236+
SortedSetStarTreeValuesIterator k2 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator("ip1");
227237
for (StarTreeDocument starDoc : actualStarTreeDocuments) {
228238
String keyword1 = null;
229239
if (starDoc.dimensions[0] != null) {
@@ -232,7 +242,11 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {
232242

233243
String keyword2 = null;
234244
if (starDoc.dimensions[1] != null) {
235-
keyword2 = k2.lookupOrd(starDoc.dimensions[1]).utf8ToString();
245+
BytesRef encoded = k2.lookupOrd(starDoc.dimensions[1]);
246+
InetAddress address = InetAddressPoint.decode(
247+
Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)
248+
);
249+
keyword2 = address.toString();
236250
}
237251
double metric = (double) starDoc.metrics[0];
238252
if (map.containsKey(keyword1 + "-" + keyword2)) {
@@ -254,21 +268,28 @@ public void testStarKeywordDocValuesWithMissingDocs() throws IOException {
254268
Document doc = new Document();
255269
doc.add(new SortedNumericDocValuesField("sndv", 1));
256270
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
271+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
272+
257273
iw.addDocument(doc);
258274
doc = new Document();
259275
doc.add(new SortedNumericDocValuesField("sndv", 1));
260276
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
277+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
261278
iw.addDocument(doc);
262279
iw.forceMerge(1);
263280
doc = new Document();
264281
doc.add(new SortedNumericDocValuesField("sndv", 2));
265282
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
266283
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
284+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
285+
267286
iw.addDocument(doc);
268287
doc = new Document();
269288
doc.add(new SortedNumericDocValuesField("sndv", 2));
270289
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
271290
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
291+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
292+
272293
iw.addDocument(doc);
273294
iw.forceMerge(1);
274295
iw.close();
@@ -340,11 +361,14 @@ public void testStarKeywordDocValuesWithMissingDocsInSegment() throws IOExceptio
340361
doc.add(new SortedNumericDocValuesField("sndv", 2));
341362
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
342363
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
364+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
343365
iw.addDocument(doc);
344366
doc = new Document();
345367
doc.add(new SortedNumericDocValuesField("sndv", 2));
346368
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
347369
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
370+
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
371+
348372
iw.addDocument(doc);
349373
iw.forceMerge(1);
350374
iw.close();
@@ -538,7 +562,7 @@ protected XContentBuilder getMapping() throws IOException {
538562
b.field("name", "keyword1");
539563
b.endObject();
540564
b.startObject();
541-
b.field("name", "keyword2");
565+
b.field("name", "ip1");
542566
b.endObject();
543567
b.endArray();
544568
b.startArray("metrics");
@@ -566,6 +590,9 @@ protected XContentBuilder getMapping() throws IOException {
566590
b.startObject("keyword2");
567591
b.field("type", "keyword");
568592
b.endObject();
593+
b.startObject("ip1");
594+
b.field("type", "ip");
595+
b.endObject();
569596
b.endObject();
570597
});
571598
}

0 commit comments

Comments
 (0)