From 7011249e3ff1f4fe719431ab198116ce1c2033b9 Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Thu, 14 Nov 2024 22:35:16 +0530 Subject: [PATCH 01/10] handle unsigned long in flush operations for star tree --- .../datacube/DimensionFactory.java | 14 +- .../datacube/NumericDimension.java | 14 ++ .../builder/OffHeapStarTreeBuilder.java | 9 +- .../utils/StarTreeDocumentsSorter.java | 24 +++- .../index/mapper/MapperBuilderProperties.java | 9 ++ .../index/mapper/NumberFieldMapper.java | 11 +- .../index/mapper/StarTreeMapper.java | 6 +- .../utils/StarTreeDocumentsSorterTests.java | 124 +++++++++++------- 8 files changed, 150 insertions(+), 61 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index e834706e2fa9d..c99ec50b9236b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -36,6 +36,7 @@ public class DimensionFactory { public static Dimension parseAndCreateDimension( String name, String type, + Boolean isUnsignedLong, Map dimensionMap, Mapper.TypeParser.ParserContext c ) { @@ -43,7 +44,7 @@ public static Dimension parseAndCreateDimension( case DateDimension.DATE: return parseAndCreateDateDimension(name, dimensionMap, c); case NumericDimension.NUMERIC: - return new NumericDimension(name); + return new NumericDimension(name, isUnsignedLong); case KEYWORD: return new KeywordDimension(name); default: @@ -53,6 +54,15 @@ public static Dimension parseAndCreateDimension( } } + public static Dimension parseAndCreateDimension( + String name, + String type, + Map dimensionMap, + Mapper.TypeParser.ParserContext c + ) { + return parseAndCreateDimension(name, type, false, dimensionMap, c); + } + public static Dimension parseAndCreateDimension( String name, Mapper.Builder builder, @@ -68,7 +78,7 @@ public static Dimension parseAndCreateDimension( case DATE: return parseAndCreateDateDimension(name, dimensionMap, c); case NUMERIC: - return new NumericDimension(name); + return new NumericDimension(name, builder.isUnsignedLong()); case KEYWORD: return new KeywordDimension(name); default: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index fe9e3d17c0047..dc3e17458bf86 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -26,10 +26,19 @@ @ExperimentalApi public class NumericDimension implements Dimension { public static final String NUMERIC = "numeric"; + public static final String IS_UNSIGNED_LONG_FIELD = "isUnsignedLong"; + private final String field; + private final Boolean isUnsignedLong; public NumericDimension(String field) { this.field = field; + isUnsignedLong = false; + } + + public NumericDimension(String field, Boolean isUnsignedLong) { + this.field = field; + this.isUnsignedLong = isUnsignedLong; } public String getField() { @@ -61,6 +70,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(CompositeDataCubeFieldType.NAME, field); builder.field(CompositeDataCubeFieldType.TYPE, NUMERIC); + builder.field(IS_UNSIGNED_LONG_FIELD, isUnsignedLong); builder.endObject(); return builder; } @@ -77,4 +87,8 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } + + public Boolean isUnsignedLong() { + return isUnsignedLong; + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 63659ef684744..dbd0e847e7c1d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -228,6 +229,7 @@ private Iterator sortAndReduceDocuments(int[] sortedDocIds, in logger.debug("Sorted doc ids array is null"); return Collections.emptyIterator(); } + List dimensionsOrder = starTreeDocumentFileManager.starTreeField.getDimensionsOrder(); try { StarTreeDocumentsSorter.sort(sortedDocIds, -1, numDocs, index -> { try { @@ -235,7 +237,7 @@ private Iterator sortAndReduceDocuments(int[] sortedDocIds, in } catch (IOException e) { throw new UncheckedIOException(e); } - }); + }, dimensionsOrder); } catch (UncheckedIOException ex) { // Unwrap UncheckedIOException and throw as IOException if (ex.getCause() != null) { @@ -308,6 +310,7 @@ public List getStarTreeDocuments() throws IOException { @Override public Long getDimensionValue(int docId, int dimensionId) throws IOException { return starTreeDocumentFileManager.getDimensionValue(docId, dimensionId); + } /** @@ -328,13 +331,15 @@ public Iterator generateStarTreeDocumentsForStarNode(int start for (int i = 0; i < numDocs; i++) { sortedDocIds[i] = startDocId + i; } + List dimensionsOrder = starTreeDocumentFileManager.starTreeField.getDimensionsOrder(); StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, index -> { try { return starTreeDocumentFileManager.readDimensions(sortedDocIds[index]); } catch (IOException e) { throw new RuntimeException(e); } - }); + }, dimensionsOrder); + // Create an iterator for aggregated documents return new Iterator() { boolean hasNext = true; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java index 7b1c63bc611ee..e641c0f223cb5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java @@ -9,7 +9,10 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.apache.lucene.util.IntroSorter; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import java.util.List; import java.util.Objects; import java.util.function.IntFunction; @@ -24,7 +27,8 @@ public static void sort( final int[] sortedDocIds, final int dimensionId, final int numDocs, - final IntFunction dimensionsReader + final IntFunction dimensionsReader, + final List dimensionsOrder ) { new IntroSorter() { private Long[] dimensions; @@ -45,19 +49,27 @@ protected void setPivot(int i) { protected int comparePivot(int j) { Long[] currentDimensions = dimensionsReader.apply(j); for (int i = dimensionId + 1; i < dimensions.length; i++) { - Long dimension = currentDimensions[i]; - if (!Objects.equals(dimensions[i], dimension)) { - if (dimensions[i] == null && dimension == null) { + Dimension dimension = dimensionsOrder.get(i); + Long dimensionValue = currentDimensions[i]; + if (!Objects.equals(dimensions[i], dimensionValue)) { + if (dimensions[i] == null && dimensionValue == null) { return 0; } - if (dimension == null) { + if (dimensionValue == null) { return -1; } if (dimensions[i] == null) { return 1; } - return Long.compare(dimensions[i], dimension); + if (dimension instanceof NumericDimension) { + NumericDimension numericDimension = (NumericDimension) dimension; + if (numericDimension.isUnsignedLong()) { + return Long.compareUnsigned(dimensions[i], dimensionValue); + } + } + return Long.compare(dimensions[i], dimensionValue); } + } return 0; } diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java index ce7b8f28b0b29..445ffbb07262e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java @@ -29,6 +29,15 @@ default Optional getSupportedDataCubeDimensionType() { return Optional.empty(); } + /** + * Indicates whether the current dimension is unsigned long. + * + * @return true if the dimension is unsigned long, false otherwise. + */ + default Boolean isUnsignedLong() { + return false; + } + /** * Indicates whether the implementation supports data cube metrics. * diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 43e975f95757b..05fff8e3fd3d6 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -177,15 +177,14 @@ public NumberFieldMapper build(BuilderContext context) { @Override public Optional getSupportedDataCubeDimensionType() { - - // unsigned long is not supported as dimension for star tree - if (type.numericType.equals(NumericType.UNSIGNED_LONG)) { - return Optional.empty(); - } - return Optional.of(DimensionType.NUMERIC); } + @Override + public Boolean isUnsignedLong() { + return type.numericType.equals(NumericType.UNSIGNED_LONG); + } + @Override public boolean isDataCubeMetricSupported() { return true; diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index 40f05a8b76755..ecab7bf11ce25 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -34,6 +34,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.index.compositeindex.datacube.NumericDimension.IS_UNSIGNED_LONG_FIELD; + /** * A field mapper for star tree fields * @@ -239,12 +241,14 @@ private Dimension getDimension(String fieldName, Object dimensionMapping, Mapper if (this.objbuilder == null || this.objbuilder.mappersBuilders == null) { String type = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.TYPE, dimensionMap); dimensionMap.remove(CompositeDataCubeFieldType.TYPE); + Boolean isUnsignedLong = (Boolean) XContentMapValues.extractValue(IS_UNSIGNED_LONG_FIELD, dimensionMap); + dimensionMap.remove(IS_UNSIGNED_LONG_FIELD); if (type == null) { throw new MapperParsingException( String.format(Locale.ROOT, "unable to parse ordered_dimensions for star tree field [%s]", fieldName) ); } - return DimensionFactory.parseAndCreateDimension(name, type, dimensionMap, context); + return DimensionFactory.parseAndCreateDimension(name, type, isUnsignedLong, dimensionMap, context); } else { Optional dimBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); if (dimBuilder.isEmpty()) { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java index b485ea1a4fe3e..c6eb45c3ec5e0 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java @@ -9,6 +9,8 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.opensearch.common.Randomness; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -24,37 +26,79 @@ * Tests for {@link StarTreeDocumentsSorter}. */ public class StarTreeDocumentsSorterTests extends OpenSearchTestCase { + private Map testData; + private List dimensionsOrder; @Before public void setUp() throws Exception { super.setUp(); testData = new HashMap<>(); - testData.put(0, new Long[] { -1L, 2L, 3L }); - testData.put(1, new Long[] { 1L, 2L, 2L }); - testData.put(2, new Long[] { -1L, -1L, 3L }); - testData.put(3, new Long[] { 1L, 2L, null }); - testData.put(4, new Long[] { 1L, null, 3L }); + + // 10 documents with 5 dimensions each + testData.put(0, new Long[] { null, 150L, 100L, 300L, null }); + testData.put(1, new Long[] { 1L, null, -9223372036854775807L, 200L, 300L }); + testData.put(2, new Long[] { 2L, -100L, -15L, 250L, null }); + testData.put(3, new Long[] { 2L, -100L, -10L, 210L, -9223372036854775807L }); + testData.put(4, new Long[] { 1L, 120L, null, null, 305L }); + testData.put(5, new Long[] { 2L, 150L, -5L, 200L, 295L }); + testData.put(6, new Long[] { 3L, 105L, null, -200L, -315L }); + testData.put(7, new Long[] { 1L, 120L, -10L, 205L, 310L }); + testData.put(8, new Long[] { null, -100L, 9223372036854775807L, 200L, -300L }); + testData.put(9, new Long[] { 2L, null, -10L, 210L, 325L }); + + dimensionsOrder = Arrays.asList( + new NumericDimension("dim1", false), // Long + new NumericDimension("dim2", true), // Unsigned Long + new NumericDimension("dim3", false), // Long + new NumericDimension("dim4", true), // Unsigned Long + new NumericDimension("dim5", false) // Long + ); } public void testSortDocumentsOffHeap_FirstDimension() { - int[] sortedDocIds = { 0, 1, 2, 3, 4 }; + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = -1; - int numDocs = 5; + int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_SecondDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 0; + int numDocs = 10; - assertArrayEquals(new int[] { 2, 0, 1, 3, 4 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + assertArrayEquals(new int[] { 6, 7, 4, 5, 0, 2, 3, 8, 1, 9 }, sortedDocIds); } public void testSortDocumentsOffHeap_ThirdDimension() { - int[] sortedDocIds = { 0, 1, 2, 3, 4 }; + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = 1; - int numDocs = 5; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + assertArrayEquals(new int[] { 1, 2, 7, 3, 9, 5, 0, 8, 6, 4 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_FourthDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 2; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + assertArrayEquals(new int[] { 8, 5, 1, 7, 3, 9, 2, 0, 6, 4 }, sortedDocIds); + } - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + public void testSortDocumentsOffHeap_FifthDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 3; + int numDocs = 10; - assertArrayEquals(new int[] { 1, 0, 2, 4, 3 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + assertArrayEquals(new int[] { 3, 6, 8, 5, 1, 4, 7, 9, 0, 2 }, sortedDocIds); } public void testSortDocumentsOffHeap_SingleElement() { @@ -62,8 +106,7 @@ public void testSortDocumentsOffHeap_SingleElement() { int dimensionId = -1; int numDocs = 1; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); assertArrayEquals(new int[] { 0 }, sortedDocIds); } @@ -72,32 +115,21 @@ public void testSortDocumentsOffHeap_EmptyArray() { int dimensionId = -1; int numDocs = 0; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); assertArrayEquals(new int[] {}, sortedDocIds); } - public void testSortDocumentsOffHeap_SecondDimensionId() { - int[] sortedDocIds = { 0, 1, 2, 3, 4 }; - int dimensionId = 0; - int numDocs = 5; - - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - - assertArrayEquals(new int[] { 2, 1, 0, 3, 4 }, sortedDocIds); - } - public void testSortDocumentsOffHeap_AllNulls() { Map testData = new HashMap<>(); - testData.put(0, new Long[] { null, null, null }); - testData.put(1, new Long[] { null, null, null }); - testData.put(2, new Long[] { null, null, null }); + testData.put(0, new Long[] { null, null, null, null, null }); + testData.put(1, new Long[] { null, null, null, null, null }); + testData.put(2, new Long[] { null, null, null, null, null }); int[] sortedDocIds = { 0, 1, 2 }; int dimensionId = -1; int numDocs = 3; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); // The order should remain unchanged as all elements are equal (null) assertArrayEquals(new int[] { 0, 1, 2 }, sortedDocIds); @@ -105,23 +137,21 @@ public void testSortDocumentsOffHeap_AllNulls() { public void testSortDocumentsOffHeap_Negatives() { Map testData = new HashMap<>(); - testData.put(0, new Long[] { -10L, 0L }); - testData.put(1, new Long[] { -9L, 0L }); - testData.put(2, new Long[] { -8L, 0L }); - testData.put(3, new Long[] { -7L, -0L }); - testData.put(4, new Long[] { -15L, -0L }); + testData.put(0, new Long[] { -10L, 0L, null, 0L, -5L }); + testData.put(1, new Long[] { -9L, 0L, null, 0L, -10L }); + testData.put(2, new Long[] { -9L, 0L, null, 0L, 15L }); + testData.put(3, new Long[] { -7L, 0L, null, 0L, -20L }); + testData.put(4, new Long[] { -15L, 0L, null, 0L, -25L }); int[] sortedDocIds = { 0, 1, 2, 3, 4 }; int dimensionId = -1; int numDocs = 5; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); - - // The order should remain unchanged as all elements are equal (null) + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); assertArrayEquals(new int[] { 4, 0, 1, 2, 3 }, sortedDocIds); } - public void testRandomSort() { + public void testTheRandomSort() { int i = 0; while (i < 10) { testRandomizedSort(); @@ -157,8 +187,14 @@ private void testRandomizedSort() { // for example to start from dimension in 0th index, we need to pass -1 to sort method int dimensionId = random.nextInt(numDimensions) - 1; + List dimensionsOrder = new ArrayList<>(); + for (int i = 0; i < numDimensions; i++) { + Boolean isUnsignedLong = random.nextBoolean(); + dimensionsOrder.add(new NumericDimension("fieldName", isUnsignedLong)); + } + // Sort using StarTreeDocumentsSorter - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i])); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); // Verify the sorting for (int i = 1; i < numDocs; i++) { @@ -166,7 +202,7 @@ private void testRandomizedSort() { Long[] curr = testData.get(sortedDocIds[i]); boolean isCorrectOrder = true; for (int j = dimensionId + 1; j < numDimensions; j++) { - int comparison = compareLongs(prev[j], curr[j]); + int comparison = compareLongs(prev[j], curr[j], ((NumericDimension) dimensionsOrder.get(j)).isUnsignedLong()); if (comparison < 0) { break; } else if (comparison > 0) { @@ -186,14 +222,14 @@ private void testRandomizedSort() { } } - private int compareLongs(Long a, Long b) { + private int compareLongs(Long a, Long b, Boolean isUnsignedLong) { if (!Objects.equals(a, b)) { if (a == null) { return 1; } else if (b == null) { return -1; } else { - return a.compareTo(b); + return isUnsignedLong ? Long.compareUnsigned(a, b) : Long.compare(a, b); } } return 0; From 1716e7fa7b7562738d58486ab7755ce86d72fceb Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Tue, 26 Nov 2024 08:41:38 +0530 Subject: [PATCH 02/10] Add Flush Flow and Metric Tests --- .../datacube/ComparatorType.java | 52 ++++ .../compositeindex/datacube/Dimension.java | 20 +- .../datacube/DimensionFactory.java | 21 +- .../datacube/DimensionType.java | 6 + .../datacube/NumericDimension.java | 14 +- .../datacube/ReadDimension.java | 15 + .../datacube/UnsignedLongDimension.java | 38 +++ .../builder/OnHeapStarTreeBuilder.java | 16 +- .../node/FixedLengthStarTreeNode.java | 15 +- .../datacube/startree/node/StarTreeNode.java | 5 +- .../utils/StarTreeDocumentsSorter.java | 21 +- .../index/mapper/MapperBuilderProperties.java | 9 - .../index/mapper/NumberFieldMapper.java | 7 +- .../index/mapper/StarTreeMapper.java | 6 +- .../search/startree/StarTreeFilter.java | 9 +- .../StarTreeDocValuesFormatTests.java | 40 ++- .../datacube/startree/StarTreeTestUtils.java | 20 +- .../startree/builder/BuilderTestsUtils.java | 2 +- .../builder/StarTreeBuildMetricTests.java | 152 ++++++++- .../StarTreeBuilderFlushFlowTests.java | 126 ++++++++ .../StarTreeBuilderMergeFlowTests.java | 293 +++++++++++++++++- .../builder/StarTreeBuilderTestCase.java | 66 ++++ .../data/StarTreeFileFormatsTests.java | 8 +- .../node/FixedLengthStarTreeNodeTests.java | 12 +- .../utils/StarTreeDocumentsSorterTests.java | 19 +- .../index/mapper/StarTreeMapperTests.java | 34 +- 26 files changed, 888 insertions(+), 138 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java new file mode 100644 index 0000000000000..ce4d20c9c228b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Represents the type of comparison to be performed on a dimension. + * + * @opensearch.experimental + */ +@ExperimentalApi +public enum ComparatorType { + LONG { + @Override + int compare(Long a, Long b) { + if (a == null && b == null) { + return 0; + } + if (b == null) { + return -1; + } + if (a == null) { + return 1; + } + return Long.compare(a, b); + } + }, + UNSIGNED_LONG { + @Override + int compare(Long a, Long b) { + if (a == null && b == null) { + return 0; + } + if (b == null) { + return -1; + } + if (a == null) { + return 1; + } + return Long.compareUnsigned(a, b); + } + }; + + abstract int compare(Long a, Long b); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java index 3d71b38881693..648002dc742dd 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -12,6 +12,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; +import java.util.Comparator; import java.util.List; import java.util.function.Consumer; @@ -34,8 +35,8 @@ public interface Dimension extends ToXContent { /** * Sets the dimension values with the consumer * - * @param value The value to be set - * @param dimSetter Consumer which sets the dimensions + * @param value The value to be set + * @param dimSetter Consumer which sets the dimensions */ void setDimensionValues(final Long value, final Consumer dimSetter); @@ -45,4 +46,19 @@ public interface Dimension extends ToXContent { List getSubDimensionNames(); DocValuesType getDocValuesType(); + + /** + * Returns the comparator type used for comparing dimension values.
+ * This determines how numeric values are compared:
+ * - ComparatorType.UNSIGNED_LONG for unsigned long values
+ * - ComparatorType.LONG for all other numeric types (DEFAULT) + */ + default ComparatorType getComparatorType() { + return ComparatorType.LONG; + } + + default Comparator comparator() { + return (a, b) -> getComparatorType().compare(a, b); + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index c99ec50b9236b..7f7d2af9af203 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -24,7 +24,6 @@ import java.util.stream.Collectors; import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS; -import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD; /** * Dimension factory class mainly used to parse and create dimension from the mappings @@ -36,7 +35,6 @@ public class DimensionFactory { public static Dimension parseAndCreateDimension( String name, String type, - Boolean isUnsignedLong, Map dimensionMap, Mapper.TypeParser.ParserContext c ) { @@ -44,8 +42,10 @@ public static Dimension parseAndCreateDimension( case DateDimension.DATE: return parseAndCreateDateDimension(name, dimensionMap, c); case NumericDimension.NUMERIC: - return new NumericDimension(name, isUnsignedLong); - case KEYWORD: + return new NumericDimension(name); + case UnsignedLongDimension.UNSIGNED_LONG: + return new UnsignedLongDimension(name); + case KeywordDimension.KEYWORD: return new KeywordDimension(name); default: throw new IllegalArgumentException( @@ -54,15 +54,6 @@ public static Dimension parseAndCreateDimension( } } - public static Dimension parseAndCreateDimension( - String name, - String type, - Map dimensionMap, - Mapper.TypeParser.ParserContext c - ) { - return parseAndCreateDimension(name, type, false, dimensionMap, c); - } - public static Dimension parseAndCreateDimension( String name, Mapper.Builder builder, @@ -78,7 +69,9 @@ public static Dimension parseAndCreateDimension( case DATE: return parseAndCreateDateDimension(name, dimensionMap, c); case NUMERIC: - return new NumericDimension(name, builder.isUnsignedLong()); + return new NumericDimension(name); + case UNSIGNED_LONG: + return new UnsignedLongDimension(name); case KEYWORD: return new KeywordDimension(name); default: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java index d327f8ca1fa1e..a307169f966a5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java @@ -23,6 +23,12 @@ public enum DimensionType { */ NUMERIC, + /** + * Represents an unsigned long dimension type. + * This is used for dimensions that contain numerical values of type unsigned long. + */ + UNSIGNED_LONG, + /** * Represents a date dimension type. * This is used for dimensions that contain date or timestamp values. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index dc3e17458bf86..50c83b11aa35c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -26,19 +26,11 @@ @ExperimentalApi public class NumericDimension implements Dimension { public static final String NUMERIC = "numeric"; - public static final String IS_UNSIGNED_LONG_FIELD = "isUnsignedLong"; - private final String field; - private final Boolean isUnsignedLong; + protected final String field; public NumericDimension(String field) { this.field = field; - isUnsignedLong = false; - } - - public NumericDimension(String field, Boolean isUnsignedLong) { - this.field = field; - this.isUnsignedLong = isUnsignedLong; } public String getField() { @@ -70,7 +62,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(CompositeDataCubeFieldType.NAME, field); builder.field(CompositeDataCubeFieldType.TYPE, NUMERIC); - builder.field(IS_UNSIGNED_LONG_FIELD, isUnsignedLong); builder.endObject(); return builder; } @@ -88,7 +79,4 @@ public int hashCode() { return Objects.hash(field); } - public Boolean isUnsignedLong() { - return isUnsignedLong; - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index 384553a8f7e06..ac1a784d8b4b5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -26,15 +26,24 @@ public class ReadDimension implements Dimension { public static final String READ = "read"; private final String field; private final DocValuesType docValuesType; + private final ComparatorType comparatorType; public ReadDimension(String field) { this.field = field; this.docValuesType = DocValuesType.SORTED_NUMERIC; + this.comparatorType = ComparatorType.LONG; } public ReadDimension(String field, DocValuesType docValuesType) { this.field = field; this.docValuesType = docValuesType; + this.comparatorType = ComparatorType.LONG; + } + + public ReadDimension(String field, DocValuesType docValuesType, ComparatorType comparatorType) { + this.field = field; + this.docValuesType = docValuesType; + this.comparatorType = comparatorType; } public String getField() { @@ -82,4 +91,10 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } + + @Override + public ComparatorType getComparatorType() { + return comparatorType; + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java new file mode 100644 index 0000000000000..9b93a4adbcfd0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; + +import java.io.IOException; + +public class UnsignedLongDimension extends NumericDimension { + + public static final String UNSIGNED_LONG = "unsigned_long"; + + public UnsignedLongDimension(String field) { + super(field); + } + + @Override + public ComparatorType getComparatorType() { + return ComparatorType.UNSIGNED_LONG; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CompositeDataCubeFieldType.NAME, field); + builder.field(CompositeDataCubeFieldType.TYPE, UNSIGNED_LONG); + builder.endObject(); + return builder; + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index c91f4c5db98bb..fac8764ed8a3e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -13,6 +13,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -22,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -271,18 +273,12 @@ public StarTreeDocument next() { */ private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) { Arrays.sort(starTreeDocuments, (o1, o2) -> { + List dimensionsOrder = starTreeField.getDimensionsOrder(); for (int i = dimensionId; i < numDimensions; i++) { if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { - if (o1.dimensions[i] == null && o2.dimensions[i] == null) { - return 0; - } - if (o1.dimensions[i] == null) { - return 1; - } - if (o2.dimensions[i] == null) { - return -1; - } - return Long.compare(o1.dimensions[i], o2.dimensions[i]); + Dimension dimension = dimensionsOrder.get(i); + Comparator comparator = dimension.comparator(); + return comparator.compare(o1.dimensions[i], o2.dimensions[i]); } } return 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index df2ce9096bfc1..8f67b611f08ca 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -8,11 +8,13 @@ package org.opensearch.index.compositeindex.datacube.startree.fileformats.node; import org.apache.lucene.store.RandomAccessInput; +import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Comparator; import java.util.Iterator; /** @@ -192,7 +194,7 @@ public StarTreeNode getChildStarNode() throws IOException { } @Override - public StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException { + public StarTreeNode getChildForDimensionValue(Long dimensionValue, Dimension dimension) throws IOException { // there will be no children for leaf nodes if (isLeaf()) { return null; @@ -200,7 +202,7 @@ public StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOExce StarTreeNode resultStarTreeNode = null; if (null != dimensionValue) { - resultStarTreeNode = binarySearchChild(dimensionValue); + resultStarTreeNode = binarySearchChild(dimensionValue, dimension); } return resultStarTreeNode; } @@ -240,7 +242,7 @@ private static FixedLengthStarTreeNode matchStarTreeNodeTypeOrNull(FixedLengthSt * @return The child node if found, null otherwise * @throws IOException If there's an error reading from the input */ - private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IOException { + private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, Dimension dimension) throws IOException { int low = firstChildId; @@ -255,14 +257,15 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IO high--; } + Comparator comparator = dimension.comparator(); while (low <= high) { int mid = low + (high - low) / 2; FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid); long midDimensionValue = midNode.getDimensionValue(); - - if (midDimensionValue == dimensionValue) { + int compare = comparator.compare(midDimensionValue, dimensionValue); + if (compare == 0) { return midNode; - } else if (midDimensionValue < dimensionValue) { + } else if (compare < 0) { low = mid + 1; } else { high = mid - 1; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index fce3e30e9ebf6..bebf7d476d43b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -9,6 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.node; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.Dimension; import java.io.IOException; import java.util.Iterator; @@ -103,11 +104,11 @@ public interface StarTreeNode { /** * Returns the child node for the given dimension value in the star-tree. * - * @param dimensionValue the dimension value + * @param dimensionValue the dimension value * @return the child node for the given dimension value or null if child is not present * @throws IOException if an I/O error occurs while retrieving the child node */ - StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException; + StarTreeNode getChildForDimensionValue(Long dimensionValue, Dimension dimension) throws IOException; /** * Returns the child star node for a node in the star-tree. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java index e641c0f223cb5..27131122d77ec 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java @@ -10,8 +10,8 @@ import org.apache.lucene.util.IntroSorter; import org.opensearch.index.compositeindex.datacube.Dimension; -import org.opensearch.index.compositeindex.datacube.NumericDimension; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.function.IntFunction; @@ -52,24 +52,9 @@ protected int comparePivot(int j) { Dimension dimension = dimensionsOrder.get(i); Long dimensionValue = currentDimensions[i]; if (!Objects.equals(dimensions[i], dimensionValue)) { - if (dimensions[i] == null && dimensionValue == null) { - return 0; - } - if (dimensionValue == null) { - return -1; - } - if (dimensions[i] == null) { - return 1; - } - if (dimension instanceof NumericDimension) { - NumericDimension numericDimension = (NumericDimension) dimension; - if (numericDimension.isUnsignedLong()) { - return Long.compareUnsigned(dimensions[i], dimensionValue); - } - } - return Long.compare(dimensions[i], dimensionValue); + Comparator comparator = dimension.comparator(); + return comparator.compare(dimensions[i], dimensionValue); } - } return 0; } diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java index 445ffbb07262e..ce7b8f28b0b29 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java @@ -29,15 +29,6 @@ default Optional getSupportedDataCubeDimensionType() { return Optional.empty(); } - /** - * Indicates whether the current dimension is unsigned long. - * - * @return true if the dimension is unsigned long, false otherwise. - */ - default Boolean isUnsignedLong() { - return false; - } - /** * Indicates whether the implementation supports data cube metrics. * diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 05fff8e3fd3d6..713e891629e02 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -93,7 +93,7 @@ import org.roaringbitmap.RoaringBitmap; /** - * A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. + * A {@link FieldMapper} for numeric types: byte, short, int, long, float, double and unsigned long. * * @opensearch.internal */ @@ -177,11 +177,10 @@ public NumberFieldMapper build(BuilderContext context) { @Override public Optional getSupportedDataCubeDimensionType() { - return Optional.of(DimensionType.NUMERIC); + return isUnsignedLong() ? Optional.of(DimensionType.UNSIGNED_LONG) : Optional.of(DimensionType.NUMERIC); } - @Override - public Boolean isUnsignedLong() { + private Boolean isUnsignedLong() { return type.numericType.equals(NumericType.UNSIGNED_LONG); } diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index ecab7bf11ce25..40f05a8b76755 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -34,8 +34,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.index.compositeindex.datacube.NumericDimension.IS_UNSIGNED_LONG_FIELD; - /** * A field mapper for star tree fields * @@ -241,14 +239,12 @@ private Dimension getDimension(String fieldName, Object dimensionMapping, Mapper if (this.objbuilder == null || this.objbuilder.mappersBuilders == null) { String type = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.TYPE, dimensionMap); dimensionMap.remove(CompositeDataCubeFieldType.TYPE); - Boolean isUnsignedLong = (Boolean) XContentMapValues.extractValue(IS_UNSIGNED_LONG_FIELD, dimensionMap); - dimensionMap.remove(IS_UNSIGNED_LONG_FIELD); if (type == null) { throw new MapperParsingException( String.format(Locale.ROOT, "unable to parse ordered_dimensions for star tree field [%s]", fieldName) ); } - return DimensionFactory.parseAndCreateDimension(name, type, isUnsignedLong, dimensionMap, context); + return DimensionFactory.parseAndCreateDimension(name, type, dimensionMap, context); } else { Optional dimBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); if (dimBuilder.isEmpty()) { diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java index f7fa210691678..0e062db0ddcf8 100644 --- a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java @@ -118,11 +118,8 @@ private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Ma DocIdSetBuilder.BulkAdder adder; Set globalRemainingPredicateColumns = null; StarTreeNode starTree = starTreeValues.getRoot(); - List dimensionNames = starTreeValues.getStarTreeField() - .getDimensionsOrder() - .stream() - .map(Dimension::getField) - .collect(Collectors.toList()); + List dimensionsOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); + List dimensionNames = dimensionsOrder.stream().map(Dimension::getField).collect(Collectors.toList()); boolean foundLeafNode = starTree.isLeaf(); assert foundLeafNode == false; // root node is never leaf Queue queue = new ArrayDeque<>(); @@ -170,7 +167,7 @@ private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Ma if (remainingPredicateColumns.contains(childDimension)) { long queryValue = queryMap.get(childDimension); // Get the query value directly from the map - StarTreeNode matchingChild = starTreeNode.getChildForDimensionValue(queryValue); + StarTreeNode matchingChild = starTreeNode.getChildForDimensionValue(queryValue, dimensionsOrder.get(dimensionId + 1)); if (matchingChild != null) { queue.add(matchingChild); foundLeafNode |= matchingChild.isLeaf(); diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index 03798c6e4ce55..93dcf3b32da07 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -57,22 +57,26 @@ public void testStarTreeDocValues() throws IOException { conf.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", 10)); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv1", 1)); doc.add(new SortedNumericDocValuesField("field1", -1)); iw.addDocument(doc); doc = new Document(); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", 10)); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv1", 1)); doc.add(new SortedNumericDocValuesField("field1", -1)); iw.addDocument(doc); doc = new Document(); iw.forceMerge(1); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", -20)); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv1", 2)); doc.add(new SortedNumericDocValuesField("field1", -2)); iw.addDocument(doc); doc = new Document(); + doc.add(new SortedNumericDocValuesField("unsignedLongDimension", -20)); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv1", 2)); doc.add(new SortedNumericDocValuesField("field1", -2)); @@ -86,35 +90,35 @@ public void testStarTreeDocValues() throws IOException { // Segment documents /** - * sndv dv field - * [1, 1, -1] - * [1, 1, -1] - * [2, 2, -2] - * [2, 2, -2] + * sndv dv unsignedLongDimension field + * [10, 1, 1, -1] + * [10, 1, 1, -1] + * [-20, 2, 2, -2] + * [-20, 2, 2, -2] */ - // Star tree docuements + // Star tree documents /** - * sndv dv | [ sum, value_count, min, max[field]] , [ sum, value_count, min, max[sndv]], doc_count - * [1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] - * [2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] - * [null, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] - * [null, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + * unsignedLongDimension sndv dv | [ sum, value_count, min, max[field]] , [ sum, value_count, min, max[sndv]], doc_count + * [10, 1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [-20, 2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + * [null, 1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [null, 2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] */ StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[4]; expectedStarTreeDocuments[0] = new StarTreeDocument( - new Long[] { 1L, 1L }, + new Long[] { 10L, 1L, 1L }, new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } ); expectedStarTreeDocuments[1] = new StarTreeDocument( - new Long[] { 2L, 2L }, + new Long[] { -20L, 2L, 2L }, new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } ); expectedStarTreeDocuments[2] = new StarTreeDocument( - new Long[] { null, 1L }, + new Long[] { null, 1L, 1L }, new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } ); expectedStarTreeDocuments[3] = new StarTreeDocument( - new Long[] { null, 2L }, + new Long[] { null, 2L, 2L }, new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } ); @@ -264,6 +268,9 @@ public static XContentBuilder getExpandedMapping() throws IOException { b.field("max_leaf_docs", 1); b.startArray("ordered_dimensions"); b.startObject(); + b.field("name", "unsignedLongDimension"); // UnsignedLongDimension + b.endObject(); + b.startObject(); b.field("name", "sndv"); b.endObject(); b.startObject(); @@ -305,6 +312,9 @@ public static XContentBuilder getExpandedMapping() throws IOException { b.startObject("field1"); b.field("type", "integer"); b.endObject(); + b.startObject("unsignedLongDimension"); + b.field("type", "unsigned_long"); + b.endObject(); b.endObject(); }); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 44e40f1db4cc8..f9dc31ffcbad3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -161,7 +161,8 @@ public static void validateFileFormats( IndexInput dataIn, IndexInput metaIn, InMemoryTreeNode rootNode, - StarTreeMetadata expectedStarTreeMetadata + StarTreeMetadata expectedStarTreeMetadata, + StarTreeField starTreeField ) throws IOException { long magicMarker = metaIn.readLong(); assertEquals(COMPOSITE_FIELD_MARKER, magicMarker); @@ -201,11 +202,13 @@ public static void validateFileFormats( if (rootNode.getChildren() != null) { sortedChildren = new ArrayList<>(rootNode.getChildren().values()); } - - if (starTreeNode.getChildDimensionId() != -1) { + int dimensionId = starTreeNode.getChildDimensionId(); + List dimensionsOrder = starTreeField.getDimensionsOrder(); + if (dimensionId != -1) { assertFalse(sortedChildren.isEmpty()); int childCount = 0; boolean childStarNodeAsserted = false; + boolean nodeWithMinusOneValueFound = false; while (expectedChildrenIterator.hasNext()) { StarTreeNode child = expectedChildrenIterator.next(); InMemoryTreeNode resultChildNode = null; @@ -220,10 +223,15 @@ public static void validateFileFormats( resultChildNode = sortedChildren.get(childCount); assertNotNull(child); assertNotNull(resultChildNode); - if (child.getStarTreeNodeType() != StarTreeNodeType.NULL.getValue()) { - assertNotNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); + if (child.getStarTreeNodeType() == StarTreeNodeType.NULL.getValue() && !nodeWithMinusOneValueFound) { + assertNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue(), dimensionsOrder.get(dimensionId))); } else { - assertNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); + if (child.getDimensionValue() == -1L) { + nodeWithMinusOneValueFound = true; + } + assertNotNull( + starTreeNode.getChildForDimensionValue(child.getDimensionValue(), dimensionsOrder.get(dimensionId)) + ); } assertStarTreeNode(child, resultChildNode); assertNotEquals(child.getStarTreeNodeType(), StarTreeNodeType.STAR.getValue()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java index 077bf0422ab50..03dd681303070 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java @@ -438,7 +438,7 @@ public static void validateStarTreeFileFormats( StarTreeDocument[] expectedStarTreeDocumentsArray = expectedStarTreeDocuments.toArray(new StarTreeDocument[0]); StarTreeTestUtils.assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocumentsArray); - validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata); + validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata, starTreeField); dataIn.close(); metaIn.close(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java index 95adae9335740..bc2f5cd0c3fd3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java @@ -29,6 +29,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; +import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; @@ -439,6 +440,130 @@ public void test_build_longMetrics() throws IOException { ); } + public void test_build_unsigned_longMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder( + "field2", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder( + "field4", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder( + "field6", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder( + "field9", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder( + "field10", + NumberFieldMapper.NumberType.UNSIGNED_LONG, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 12L, 10L, randomLong(), 8L, -1L }); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Long[] { -2L, -9223372036854775808L, randomLong(), 12L, 10L } + ); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 14L, 12L, randomLong(), 6L, 24L }); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Long[] { 9L, 4L, randomLong(), -9223372036854775806L, 12L } + ); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 11L, 16L, randomLong(), 8L, 13L }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = (Long) starTreeDocuments[i].metrics[0]; + long metric2 = (Long) starTreeDocuments[i].metrics[1]; + long metric3 = (Long) starTreeDocuments[i].metrics[2]; + long metric4 = (Long) starTreeDocuments[i].metrics[3]; + long metric5 = (Long) starTreeDocuments[i].metrics[4]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIteratorForUnsignedLong().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIteratorForUnsignedLong().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIteratorForUnsignedLong() + ); + } + public void test_build_multipleStarTrees() throws IOException { int noOfStarTreeDocuments = 5; @@ -654,9 +779,32 @@ public void test_build_multipleStarTrees() throws IOException { IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); + List dimensionsOrder1 = List.of( + new NumericDimension("field1"), + new NumericDimension("field3"), + new NumericDimension("field5"), + new NumericDimension("field8") + ); + List dimensionsOrder2 = List.of( + new NumericDimension("fieldC"), + new NumericDimension("fieldB"), + new NumericDimension("fieldL") + ); + StarTreeField compositeField1 = new StarTreeField( + "test", + dimensionsOrder1, + metrics, + new StarTreeFieldConfiguration(1, Set.of(), getBuildMode()) + ); + StarTreeField compositeField2 = new StarTreeField( + "test", + dimensionsOrder2, + metrics, + new StarTreeFieldConfiguration(1, Set.of(), getBuildMode()) + ); - validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata); - validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2); + validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata, compositeField1); + validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2, compositeField2); dataIn.close(); metaIn.close(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index 440268f1f803c..b4c0c680373ee 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -24,6 +24,7 @@ import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; @@ -258,6 +259,119 @@ public void testFlushFlowDimsReverse() throws IOException { ); } + public void testFlushFlowWithUnsignedLongDimensions() throws IOException { + List dimList = List.of(0L, -1L, 9223372036854775806L, 4987L, -9223372036854775807L); + List docsWithField = List.of(0, 1, 3, 4, 5); + List dimList2 = List.of(0L, -1L, 2L, 9223372036854775806L, 4987L, -9223372036854775807L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithUnsignedLongField(); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + + writeState = getWriteState(6, writeState.segmentInfo.getId()); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + + /* + Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [0, 0] | [0.0, 1] + [4987, 4987] | [40.0, 1] + [9223372036854775806, 9223372036854775806] | [30.0, 1] + [-9223372036854775807, -9223372036854775807] | [50.0, 1] + [-1, -1] | [10.0, 1] + [null, 2] | [20.0, 1] + */ + Object[][] expectedSortedDimensions = { + { 0L, 0L }, + { 4987L, 4987L }, + { 9223372036854775806L, 9223372036854775806L }, + { -9223372036854775807L, -9223372036854775807L }, + { -1L, -1L }, + { null, 2L } }; + + double[] expectedSumMetrics = { 0.0, 40.0, 30.0, 50.0, 10.0, 20.0 }; + long expectedCountMetric = 1L; + + int count = 0; + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + if (count < 6) { + assertEquals(expectedSumMetrics[count], starTreeDocument.metrics[0]); + assertEquals(expectedCountMetric, starTreeDocument.metrics[1]); + + Long dim1 = starTreeDocument.dimensions[0]; + Long dim2 = starTreeDocument.dimensions[1]; + assertEquals(expectedSortedDimensions[count][0], dim1); + assertEquals(expectedSortedDimensions[count][1], dim2); + } + count++; + } + assertEquals(13, count); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", DocValuesType.SORTED_NUMERIC); + docValues.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + docValues, + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + public void testFlushFlowBuild() throws IOException { List dimList = new ArrayList<>(100); List docsWithField = new ArrayList<>(100); @@ -531,6 +645,18 @@ private StarTreeField getStarTreeFieldWithMultipleMetrics() { return new StarTreeField("sf", dims, metrics, c); } + private StarTreeField getStarTreeFieldWithUnsignedLongField() { + Dimension d1 = new UnsignedLongDimension("field1"); + Dimension d2 = new UnsignedLongDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); + Metric m3 = new Metric("field2", List.of(MetricStat.AVG)); + List dims = List.of(d1, d2); + List metrics = List.of(m1, m2, m3); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + private StarTreeField getStarTreeFieldWithKeywordField() { Dimension d1 = new KeywordDimension("field1"); Dimension d2 = new KeywordDimension("field3"); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index be16961e781db..90b283f235168 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -18,10 +18,12 @@ import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.CompositeIndexConstants; +import org.opensearch.index.compositeindex.datacube.ComparatorType; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.ReadDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; @@ -94,13 +96,6 @@ public void testMergeFlow() throws IOException { docsWithField4.add(i); } - List dimList5 = new ArrayList<>(1000); - List docsWithField5 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList5.add((long) i); - docsWithField5.add(i); - } - List metricsList = new ArrayList<>(1000); List metricsWithField = new ArrayList<>(1000); for (int i = 0; i < 1000; i++) { @@ -119,7 +114,6 @@ public void testMergeFlow() throws IOException { Dimension d2 = new NumericDimension("field3"); Dimension d3 = new NumericDimension("field5"); Dimension d4 = new NumericDimension("field8"); - // Dimension d5 = new NumericDimension("field5"); Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); List dims = List.of(d1, d2, d3, d4); @@ -216,11 +210,203 @@ public void testMergeFlow() throws IOException { ... [999, 999, 999, 999] | [19980.0] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); - assertEquals(2L, starTreeDocument.metrics[1]); + if (count < 1000) { + assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); + assertEquals(2L, starTreeDocument.metrics[2]); + } + count++; } + // Validate the star tree structure + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 1000, + compositeField.getStarTreeConfig().maxLeafDocs(), + 132165 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowForUnsignedLong() throws IOException { + int numDocs = 1000; + List dimList1 = new ArrayList<>(numDocs); + List docsWithField1 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList1.add((long) (i % 2 == 0 ? i : -i)); + docsWithField1.add(i); + } + + List dimList2 = new ArrayList<>(numDocs); + List docsWithField2 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList2.add((long) (i % 2 == 0 ? i : -i)); + docsWithField2.add(i); + } + + List dimList3 = new ArrayList<>(numDocs); + List docsWithField3 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList3.add((long) (i % 2 == 0 ? i : -i)); + docsWithField3.add(i); + } + + List dimList4 = new ArrayList<>(numDocs); + List docsWithField4 = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + dimList4.add((long) (i % 2 == 0 ? i : -i)); + docsWithField4.add(i); + } + + List metricsList = new ArrayList<>(1000); + List metricsWithField = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + + List metricsListValueCount = new ArrayList<>(1000); + List metricsWithFieldValueCount = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsListValueCount.add((long) i); + metricsWithFieldValueCount.add(i); + } + + Dimension d1 = new ReadDimension("field1", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); + Dimension d2 = new ReadDimension("field3", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); + Dimension d3 = new ReadDimension("field5", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); + Dimension d4 = new ReadDimension("field8", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); + + Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); + Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); + + List dims = List.of(d1, d2, d3, d4); + List metrics = List.of(m1, m2); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + compositeField = new StarTreeField("sf", dims, metrics, c); + SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); + SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(d4sndv) + ); + + Map> metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> new SortedNumericStarTreeValuesIterator(m1sndv), + "sf_field2_value_count_metric", + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), + "sf__doc_count_doc_count_metric", + () -> new SortedNumericStarTreeValuesIterator(m2sndv) + ); + + StarTreeValues starTreeValues = new StarTreeValues( + compositeField, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + getAttributes(1000), + null + ); + + SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues f2d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues f2d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues f2d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); + SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); + Map> f2dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) + ); + + Map> f2metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), + "sf_field2_value_count_metric", + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), + "sf__doc_count_doc_count_metric", + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) + ); + StarTreeValues starTreeValues2 = new StarTreeValues( + compositeField, + null, + f2dimDocIdSetIterators, + f2metricDocIdSetIterators, + getAttributes(1000), + null + ); + + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [0, 0, 0, 0] | [0.0, 2] + [-1, -1, -1, -1] | [20.0, 2] + [2, 2, 2, 2] | [40.0, 2] + [-3, -3, -3, -3] | [60.0, 2] + [4, 4, 4, 4] | [80.0, 2] + [-5, -5, -5, -5] | [100.0, 2] + ... + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + List actualDimensionValues = new ArrayList<>(numDocs); + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + if (count < 1000) { + actualDimensionValues.add(starTreeDocument.dimensions[0]); + } + count++; + } + + List expectedDimensionValues = new ArrayList<>(1000); + for (int i = 0; i < numDocs; i++) { + if (i <= 499) { + expectedDimensionValues.add((long) i * 2); // Positive even numbers + } else { + expectedDimensionValues.add((long) -(numDocs - i) * 2 + 1); // Negative odd numbers in decreasing order + } + } + assertEquals(expectedDimensionValues, actualDimensionValues); // Validate the star tree structure validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); @@ -365,6 +551,93 @@ public void testMergeFlowWithSum() throws IOException { ); } + public void testMergeFlowForUnsignedLongWithSum() throws IOException { + List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); + List docsWithField = List.of(0, 1, 3, 4, 5, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + compositeField = getStarTreeFieldForUnsignedLong(MetricStat.SUM); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] + * [0, 0] | [0.0] + * [1, 1] | [20.0] + * [3, 3] | [60.0] + * [4, 4] | [80.0] + * [5, 5] | [100.0] + * [null, 2] | [40.0] + * ------------------ We only take non-star docs + * [6,-1] | [120.0] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + System.out.println(starTreeDocument); + if (count <= 6) { + assertEquals( + starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 * 10.0 : 40.0, + starTreeDocument.metrics[0] + ); + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + public void testMergeFlowWithCount() throws IOException { List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); List docsWithField = List.of(0, 1, 3, 4, 5, 6); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java index 9c9beaea4f52c..0f873cdc55238 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java @@ -25,6 +25,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Numbers; import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; @@ -36,6 +37,7 @@ import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; @@ -274,6 +276,16 @@ protected StarTreeField getStarTreeField(MetricStat count) { return new StarTreeField("sf", dims, metrics, c); } + protected StarTreeField getStarTreeFieldForUnsignedLong(MetricStat count) { + Dimension d1 = new UnsignedLongDimension("field1"); + Dimension d2 = new UnsignedLongDimension("field3"); + Metric m1 = new Metric("field2", List.of(count)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + protected StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boolean includeDocCountMetric) { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); @@ -324,6 +336,60 @@ protected static List getExpectedStarTreeDocumentIterator() { ); } + protected static List getExpectedStarTreeDocumentIteratorForUnsignedLong() { + return List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, Numbers.unsignedLongToDouble(-1), 2L }), + new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { + 20.0 + Numbers.unsignedLongToDouble(-2L), + 28.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 3L, + 6.0, + 24.0, + 3L } + ), + new StarTreeDocument( + new Long[] { null, 4L, 2L, 1L }, + new Object[] { + 20.0 + Numbers.unsignedLongToDouble(-2L), + 28.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 3L, + 6.0, + 24.0, + 3L } + ), + new StarTreeDocument( + new Long[] { null, 4L, 3L, 4L }, + new Object[] { 21.0, 14.0, 2L, 8.0, Numbers.unsignedLongToDouble(-1), 2L } + ), + new StarTreeDocument( + new Long[] { null, 4L, null, 1L }, + new Object[] { + 20.0 + Numbers.unsignedLongToDouble(-2L), + 28.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 3L, + 6.0, + 24.0, + 3L } + ), + new StarTreeDocument( + new Long[] { null, 4L, null, 4L }, + new Object[] { 21.0, 14.0, 2L, 8.0, Numbers.unsignedLongToDouble(-1), 2L } + ), + new StarTreeDocument( + new Long[] { null, 4L, null, null }, + new Object[] { + 46.0 + Numbers.unsignedLongToDouble(-2L), + 42.0 + Numbers.unsignedLongToDouble(-9223372036854775808L), + 5L, + 6.0, + Numbers.unsignedLongToDouble(-1), + 5L } + ) + ); + } + protected long getLongFromDouble(double value) { return NumericUtils.doubleToSortableLong(value); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java index 4d2aa5eaf78cf..7328562d03dad 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; @@ -80,7 +81,7 @@ public void test_StarTreeNode() throws IOException { StarTreeNode child = childrenIterator.next(); if (child.getStarTreeNodeType() == StarTreeNodeType.DEFAULT.getValue()) { assertStarTreeNode( - starTreeNode.getChildForDimensionValue(child.getDimensionValue()), + starTreeNode.getChildForDimensionValue(child.getDimensionValue(), new NumericDimension(("field"))), inMemoryTreeNodeMap.get(child.getDimensionValue()) ); assertNull(starTreeNode.getChildStarNode()); @@ -121,7 +122,10 @@ public void test_starTreeSearch() throws IOException { for (int i = 0; i < maxLevels - 1; i++) { InMemoryTreeNode randomChildNode = randomFrom(inMemoryTreeNode.getChildren().values()); - StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue(randomChildNode.getDimensionValue()); + StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue( + randomChildNode.getDimensionValue(), + new NumericDimension("field") + ); assertNotNull(randomStarTreeChildNode); assertStarTreeNode(randomStarTreeChildNode, randomChildNode); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java index 08815d5ef55f5..3f2c6c89f45dd 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; @@ -141,7 +142,10 @@ public void testGetStarTreeNodeType() throws IOException { public void testGetChildForDimensionValue() throws IOException { long dimensionValue = randomIntBetween(-1, node.getChildren().size() - 3); - FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(dimensionValue); + FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue( + dimensionValue, + new NumericDimension("field") + ); assertNotNull(childNode); assertEquals(dimensionValue, childNode.getDimensionValue()); } @@ -185,13 +189,13 @@ public void testGetChildForStarNode() throws IOException { } public void testGetChildForNullNode() throws IOException { - FixedLengthStarTreeNode nullNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(null); + FixedLengthStarTreeNode nullNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(null, null); assertNull(nullNode); } public void testGetChildForInvalidDimensionValue() throws IOException { long invalidDimensionValue = Long.MAX_VALUE; - assertNull(starTreeNode.getChildForDimensionValue(invalidDimensionValue)); + assertNull(starTreeNode.getChildForDimensionValue(invalidDimensionValue, new NumericDimension("field"))); } public void testOnlyRootNodePresent() throws IOException { @@ -219,7 +223,7 @@ public void testOnlyRootNodePresent() throws IOException { FixedLengthStarTreeNode starTreeNode = (FixedLengthStarTreeNode) StarTreeFactory.createStarTree(dataIn, starTreeMetadata); assertEquals(starTreeNode.getNumChildren(), 0); - assertNull(starTreeNode.getChildForDimensionValue(randomLong())); + assertNull(starTreeNode.getChildForDimensionValue(randomLong(), new NumericDimension("field"))); assertThrows(IllegalArgumentException.class, () -> starTreeNode.getChildrenIterator().next()); assertThrows(UnsupportedOperationException.class, () -> starTreeNode.getChildrenIterator().remove()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java index c6eb45c3ec5e0..b7fe985fc1d01 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java @@ -11,6 +11,7 @@ import org.opensearch.common.Randomness; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -48,11 +49,11 @@ public void setUp() throws Exception { testData.put(9, new Long[] { 2L, null, -10L, 210L, 325L }); dimensionsOrder = Arrays.asList( - new NumericDimension("dim1", false), // Long - new NumericDimension("dim2", true), // Unsigned Long - new NumericDimension("dim3", false), // Long - new NumericDimension("dim4", true), // Unsigned Long - new NumericDimension("dim5", false) // Long + new NumericDimension("dim1"), // Long + new UnsignedLongDimension("dim2"), // Unsigned Long + new NumericDimension("dim3"), // Long + new UnsignedLongDimension("dim4"), // Unsigned Long + new NumericDimension("dim5") // Long ); } @@ -190,7 +191,9 @@ private void testRandomizedSort() { List dimensionsOrder = new ArrayList<>(); for (int i = 0; i < numDimensions; i++) { Boolean isUnsignedLong = random.nextBoolean(); - dimensionsOrder.add(new NumericDimension("fieldName", isUnsignedLong)); + + if (isUnsignedLong) dimensionsOrder.add(new NumericDimension("fieldName")); + else dimensionsOrder.add(new UnsignedLongDimension("fieldName")); } // Sort using StarTreeDocumentsSorter @@ -202,7 +205,9 @@ private void testRandomizedSort() { Long[] curr = testData.get(sortedDocIds[i]); boolean isCorrectOrder = true; for (int j = dimensionId + 1; j < numDimensions; j++) { - int comparison = compareLongs(prev[j], curr[j], ((NumericDimension) dimensionsOrder.get(j)).isUnsignedLong()); + int comparison = -1; + if (dimensionsOrder.get(j) instanceof UnsignedLongDimension) comparison = compareLongs(prev[j], curr[j], true); + else comparison = compareLongs(prev[j], curr[j], false); if (comparison < 0) { break; } else if (comparison > 0) { diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 8ec34b3eb660c..dad4652361c67 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -25,6 +25,7 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.ReadDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; @@ -77,7 +78,7 @@ public void testValidStarTree() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals(2, starTreeFieldType.getDimensions().size()); + assertEquals(3, starTreeFieldType.getDimensions().size()); assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); @@ -89,6 +90,11 @@ public void testValidStarTree() throws IOException { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertTrue(starTreeFieldType.getDimensions().get(1) instanceof NumericDimension); + + assertEquals("unsignedLongDimension", starTreeFieldType.getDimensions().get(2).getField()); + assertTrue(starTreeFieldType.getDimensions().get(2) instanceof UnsignedLongDimension); + assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -154,6 +160,11 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(new DateTimeUnitAdapter(expectedTimeUnits.get(i)), dateDim.getIntervals().get(i)); } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertTrue(starTreeFieldType.getDimensions().get(1) instanceof NumericDimension); + + assertEquals("unsignedLongDimension", starTreeFieldType.getDimensions().get(2).getField()); + assertTrue(starTreeFieldType.getDimensions().get(2) instanceof UnsignedLongDimension); + assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); // Assert AVG gets added when both of its base metrics is already present @@ -162,7 +173,7 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status", "unsignedLongDimension")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -521,6 +532,7 @@ public void testStarTreeField() { DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); NumericDimension n1 = new NumericDimension("numeric"); NumericDimension n2 = new NumericDimension("name1"); + UnsignedLongDimension n3 = new UnsignedLongDimension("name2"); List metrics = List.of(metric1); List dims = List.of(d1, n2); @@ -534,6 +546,11 @@ public void testStarTreeField() { StarTreeField field2 = new StarTreeField("starTree", dims, metrics, config); assertEquals(field1, field2); + List dims1 = List.of(d1, n1, n2, n3); + StarTreeField field3 = new StarTreeField("starTree", dims1, metrics, config); + StarTreeField field4 = new StarTreeField("starTree", dims1, metrics, config); + assertEquals(field3, field4); + dims = List.of(d1, n2, n1); field2 = new StarTreeField("starTree", dims, metrics, config); assertNotEquals(field1, field2); @@ -650,6 +667,9 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.startObject(); b.field("name", dim); b.endObject(); + b.startObject(); + b.field("name", "unsignedLongDimension"); // UnsignedLongDimension + b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -675,6 +695,9 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.startObject("keyword1"); b.field("type", "keyword"); b.endObject(); + b.startObject("unsignedLongDimension"); + b.field("type", "unsigned_long"); + b.endObject(); b.endObject(); }); } @@ -741,6 +764,7 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) { b.value("@timestamp"); b.value("status"); + b.value("unsignedLongDimension"); } b.endArray(); b.startObject("date_dimension"); @@ -754,6 +778,9 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.startObject(); b.field("name", dim); b.endObject(); + b.startObject(); + b.field("name", "unsignedLongDimension"); // UnsignedLongDimension + b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -779,6 +806,9 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.startObject("keyword1"); b.field("type", "keyword"); b.endObject(); + b.startObject("unsignedLongDimension"); + b.field("type", "unsigned_long"); + b.endObject(); b.endObject(); }); } From 93882dfe96f65172e64e39df4d1c7ce683532f2e Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Mon, 16 Dec 2024 14:17:37 +0530 Subject: [PATCH 03/10] Handle unsigned long in merge operations in StarTree --- .../Composite912DocValuesReader.java | 7 +- .../compositeindex/datacube/Dimension.java | 14 ++-- ...aratorType.java => DimensionDataType.java} | 2 +- .../datacube/NumericDimension.java | 2 +- .../datacube/ReadDimension.java | 14 ++-- .../datacube/UnsignedLongDimension.java | 6 +- .../builder/OffHeapStarTreeBuilder.java | 7 +- .../builder/OnHeapStarTreeBuilder.java | 29 ++++--- .../startree/fileformats/StarTreeWriter.java | 5 +- .../fileformats/meta/DimensionConfig.java | 33 ++++++++ .../fileformats/meta/StarTreeMetadata.java | 75 ++++++++++++------- .../meta/StarTreeMetadataWriter.java | 30 +++++++- .../startree/index/StarTreeValues.java | 7 +- .../utils/StarTreeDocumentsSorter.java | 16 ++-- .../StarTreeDocValuesFormatTests.java | 2 +- .../startree/builder/BuilderTestsUtils.java | 5 +- .../builder/StarTreeBuildMetricTests.java | 22 +++--- .../StarTreeBuilderFlushFlowTests.java | 32 ++++---- .../StarTreeBuilderMergeFlowTests.java | 71 +++++++++--------- .../builder/StarTreeBuilderTestCase.java | 11 +-- .../meta/StarTreeMetadataTests.java | 10 +-- 21 files changed, 250 insertions(+), 150 deletions(-) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/{ComparatorType.java => DimensionDataType.java} (97%) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java index 38d3f4867e89b..021e2655185e1 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java @@ -33,6 +33,7 @@ import org.opensearch.index.compositeindex.CompositeIndexMetadata; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -156,15 +157,15 @@ public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState compositeIndexInputMap.put(compositeFieldName, starTreeIndexInput); compositeIndexMetadataMap.put(compositeFieldName, starTreeMetadata); - Map dimensionFieldToDocValuesMap = starTreeMetadata.getDimensionFields(); + Map dimensionFieldToDocValuesMap = starTreeMetadata.getDimensionFields(); // generating star tree unique fields (fully qualified name for dimension and metrics) - for (Map.Entry dimensionEntry : dimensionFieldToDocValuesMap.entrySet()) { + for (Map.Entry dimensionEntry : dimensionFieldToDocValuesMap.entrySet()) { String dimName = fullyQualifiedFieldNameForStarTreeDimensionsDocValues( compositeFieldName, dimensionEntry.getKey() ); fields.add(dimName); - dimensionFieldTypeMap.put(dimName, dimensionEntry.getValue()); + dimensionFieldTypeMap.put(dimName, dimensionEntry.getValue().getDocValuesType()); } // adding metric fields for (Metric metric : starTreeMetadata.getMetrics()) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java index 648002dc742dd..366a979946e5f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -48,17 +48,17 @@ public interface Dimension extends ToXContent { DocValuesType getDocValuesType(); /** - * Returns the comparator type used for comparing dimension values.
- * This determines how numeric values are compared:
- * - ComparatorType.UNSIGNED_LONG for unsigned long values
- * - ComparatorType.LONG for all other numeric types (DEFAULT) + * Returns the dimensionDataType used for comparing and parsing dimension values.
+ * This determines how numeric values are compared and parsed:
+ * - DimensionDataType.UNSIGNED_LONG for unsigned long values
+ * - DimensionDataType.LONG for all other numeric types (DEFAULT) */ - default ComparatorType getComparatorType() { - return ComparatorType.LONG; + default DimensionDataType getDimensionDataType() { + return DimensionDataType.LONG; } default Comparator comparator() { - return (a, b) -> getComparatorType().compare(a, b); + return (a, b) -> getDimensionDataType().compare(a, b); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java similarity index 97% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java index ce4d20c9c228b..bd9d015ef3dd5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ComparatorType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionDataType.java @@ -16,7 +16,7 @@ * @opensearch.experimental */ @ExperimentalApi -public enum ComparatorType { +public enum DimensionDataType { LONG { @Override int compare(Long a, Long b) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index 50c83b11aa35c..cd0419080332a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -27,7 +27,7 @@ public class NumericDimension implements Dimension { public static final String NUMERIC = "numeric"; - protected final String field; + private final String field; public NumericDimension(String field) { this.field = field; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index ac1a784d8b4b5..5a791188982ce 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -26,24 +26,24 @@ public class ReadDimension implements Dimension { public static final String READ = "read"; private final String field; private final DocValuesType docValuesType; - private final ComparatorType comparatorType; + private final DimensionDataType dimensionDataType; public ReadDimension(String field) { this.field = field; this.docValuesType = DocValuesType.SORTED_NUMERIC; - this.comparatorType = ComparatorType.LONG; + this.dimensionDataType = DimensionDataType.LONG; } public ReadDimension(String field, DocValuesType docValuesType) { this.field = field; this.docValuesType = docValuesType; - this.comparatorType = ComparatorType.LONG; + this.dimensionDataType = DimensionDataType.LONG; } - public ReadDimension(String field, DocValuesType docValuesType, ComparatorType comparatorType) { + public ReadDimension(String field, DocValuesType docValuesType, DimensionDataType dimensionDataType) { this.field = field; this.docValuesType = docValuesType; - this.comparatorType = comparatorType; + this.dimensionDataType = dimensionDataType; } public String getField() { @@ -93,8 +93,8 @@ public int hashCode() { } @Override - public ComparatorType getComparatorType() { - return comparatorType; + public DimensionDataType getDimensionDataType() { + return dimensionDataType; } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java index 9b93a4adbcfd0..bc11a14d1c07d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/UnsignedLongDimension.java @@ -22,14 +22,14 @@ public UnsignedLongDimension(String field) { } @Override - public ComparatorType getComparatorType() { - return ComparatorType.UNSIGNED_LONG; + public DimensionDataType getDimensionDataType() { + return DimensionDataType.UNSIGNED_LONG; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CompositeDataCubeFieldType.NAME, field); + builder.field(CompositeDataCubeFieldType.NAME, getField()); builder.field(CompositeDataCubeFieldType.TYPE, UNSIGNED_LONG); builder.endObject(); return builder; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index dbd0e847e7c1d..03b462a13ce1a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -17,7 +17,6 @@ import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -229,7 +228,6 @@ private Iterator sortAndReduceDocuments(int[] sortedDocIds, in logger.debug("Sorted doc ids array is null"); return Collections.emptyIterator(); } - List dimensionsOrder = starTreeDocumentFileManager.starTreeField.getDimensionsOrder(); try { StarTreeDocumentsSorter.sort(sortedDocIds, -1, numDocs, index -> { try { @@ -237,7 +235,7 @@ private Iterator sortAndReduceDocuments(int[] sortedDocIds, in } catch (IOException e) { throw new UncheckedIOException(e); } - }, dimensionsOrder); + }, starTreeDocumentFileManager.starTreeField.getDimensionsOrder()); } catch (UncheckedIOException ex) { // Unwrap UncheckedIOException and throw as IOException if (ex.getCause() != null) { @@ -331,14 +329,13 @@ public Iterator generateStarTreeDocumentsForStarNode(int start for (int i = 0; i < numDocs; i++) { sortedDocIds[i] = startDocId + i; } - List dimensionsOrder = starTreeDocumentFileManager.starTreeField.getDimensionsOrder(); StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, index -> { try { return starTreeDocumentFileManager.readDimensions(sortedDocIds[index]); } catch (IOException e) { throw new RuntimeException(e); } - }, dimensionsOrder); + }, starTreeDocumentFileManager.starTreeField.getDimensionsOrder()); // Create an iterator for aggregated documents return new Iterator() { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index fac8764ed8a3e..e4dc561d102a1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -44,8 +43,8 @@ public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { /** * Constructor for OnHeapStarTreeBuilder * - * @param metaOut an index output to write star-tree metadata - * @param dataOut an index output to write star-tree data + * @param metaOut an index output to write star-tree metadata + * @param dataOut an index output to write star-tree data * @param starTreeField star-tree field * @param segmentWriteState segment write state * @param mapperService helps with the numeric type of field @@ -84,9 +83,8 @@ public Long getDimensionValue(int docId, int dimensionId) { * Sorts and aggregates all the documents of the segment based on dimension and metrics configuration * * @param dimensionReaders List of docValues readers to read dimensions from the segment - * @param metricReaders List of docValues readers to read metrics from the segment + * @param metricReaders List of docValues readers to read metrics from the segment * @return Iterator of star-tree documents - * */ @Override public Iterator sortAndAggregateSegmentDocuments( @@ -269,16 +267,23 @@ public StarTreeDocument next() { * Sorts the star-tree documents from the given dimension id * * @param starTreeDocuments star-tree documents - * @param dimensionId id of the dimension + * @param dimensionId id of the dimension */ private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) { - Arrays.sort(starTreeDocuments, (o1, o2) -> { + Arrays.sort(starTreeDocuments, (doc1, doc2) -> { List dimensionsOrder = starTreeField.getDimensionsOrder(); - for (int i = dimensionId; i < numDimensions; i++) { - if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { - Dimension dimension = dimensionsOrder.get(i); - Comparator comparator = dimension.comparator(); - return comparator.compare(o1.dimensions[i], o2.dimensions[i]); + int totalDimensions = dimensionsOrder.size(); + + int docDimensionIndex = dimensionId; + for (int i = dimensionId; i < totalDimensions; i++) { + Dimension dimension = dimensionsOrder.get(i); + int subDimensionsToProcess = dimension.getNumSubDimensions(); + while (subDimensionsToProcess > 0) { + if (!Objects.equals(doc1.dimensions[docDimensionIndex], doc2.dimensions[docDimensionIndex])) { + return dimension.comparator().compare(doc1.dimensions[docDimensionIndex], doc2.dimensions[docDimensionIndex]); + } + docDimensionIndex++; + subDimensionsToProcess--; } } return 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java index e5890be3ccb5b..e888235f60ee2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java @@ -27,8 +27,11 @@ public class StarTreeWriter { /** Initial version for the star tree writer */ public static final int VERSION_START = 0; + /** Version for the star tree writer with updated metadata which handles unsigned long */ + public static final int VERSION_DIMENSION_DATA_TYPE = 1; + /** Current version for the star tree writer */ - public static final int VERSION_CURRENT = VERSION_START; + public static final int VERSION_CURRENT = VERSION_DIMENSION_DATA_TYPE; public StarTreeWriter() {} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java new file mode 100644 index 0000000000000..834f82d167ad5 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/DimensionConfig.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.fileformats.meta; + +import org.apache.lucene.index.DocValuesType; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; + +@ExperimentalApi +public class DimensionConfig { + + private final DocValuesType docValuesType; + private final DimensionDataType dimensionDataType; + + public DimensionConfig(DocValuesType docValuesType, DimensionDataType dimensionDataType) { + this.docValuesType = docValuesType; + this.dimensionDataType = dimensionDataType; + } + + public DocValuesType getDocValuesType() { + return docValuesType; + } + + public DimensionDataType getDimensionDataType() { + return dimensionDataType; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index 57e47b1a5b9d9..f7dad17ef623e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -14,9 +14,11 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.mapper.CompositeMappedFieldType; import java.io.IOException; @@ -63,10 +65,10 @@ public class StarTreeMetadata extends CompositeIndexMetadata { private final String starTreeFieldType; /** - * Map of dimension fields to their associated DocValuesType.Insertion order needs to be maintained + * Map of dimension fields to their associated DocValuesType. Insertion order needs to be maintained * as it dictates dimensionSplitOrder */ - LinkedHashMap dimensionFieldsToDocValuesMap; + LinkedHashMap dimensionFieldToDimensionConfigMap; /** * List of metrics, containing field names and associated metric statistics. @@ -114,7 +116,7 @@ public class StarTreeMetadata extends CompositeIndexMetadata { * @param metaIn an index input to read star-tree meta * @param compositeFieldName name of the composite field. Here, name of the star-tree field. * @param compositeFieldType type of the composite field. Here, STAR_TREE field. - * @param version The version of the star tree stored in the segments. + * @param version The version of the star tree stored in the segments. * @throws IOException if unable to read star-tree metadata from the file */ public StarTreeMetadata( @@ -130,7 +132,7 @@ public StarTreeMetadata( this.starTreeFieldType = this.getCompositeFieldType().getName(); this.version = version; this.numberOfNodes = readNumberOfNodes(); - this.dimensionFieldsToDocValuesMap = readStarTreeDimensions(); + this.dimensionFieldToDimensionConfigMap = readStarTreeDimensions(); this.metrics = readMetricEntries(); this.segmentAggregatedDocCount = readSegmentAggregatedDocCount(); this.starTreeDocCount = readStarTreeDocCount(); @@ -149,19 +151,19 @@ public StarTreeMetadata( * A star tree metadata constructor to initialize star tree metadata. * Used for testing. * - * @param meta an index input to read star-tree meta - * @param compositeFieldName name of the composite field. Here, name of the star-tree field. - * @param compositeFieldType type of the composite field. Here, STAR_TREE field. - * @param version The version of the star tree stored in the segments. - * @param dimensionFieldsToDocValuesMap map of dimensionFields to docValues - * @param metrics list of metric entries - * @param segmentAggregatedDocCount segment aggregated doc count - * @param starTreeDocCount the total number of star tree documents for the segment - * @param maxLeafDocs max leaf docs - * @param skipStarNodeCreationInDims set of dimensions to skip star node creation - * @param starTreeBuildMode star tree build mode - * @param dataStartFilePointer star file pointer to the associated star tree data in (.cid) file - * @param dataLength length of the corresponding star-tree data in (.cid) file + * @param meta an index input to read star-tree meta + * @param compositeFieldName name of the composite field. Here, name of the star-tree field. + * @param compositeFieldType type of the composite field. Here, STAR_TREE field. + * @param version The version of the star tree stored in the segments. + * @param dimensionFieldToDimensionConfigMap map of dimensionFields to Dimension config + * @param metrics list of metric entries + * @param segmentAggregatedDocCount segment aggregated doc count + * @param starTreeDocCount the total number of star tree documents for the segment + * @param maxLeafDocs max leaf docs + * @param skipStarNodeCreationInDims set of dimensions to skip star node creation + * @param starTreeBuildMode star tree build mode + * @param dataStartFilePointer star file pointer to the associated star tree data in (.cid) file + * @param dataLength length of the corresponding star-tree data in (.cid) file */ public StarTreeMetadata( String compositeFieldName, @@ -169,7 +171,7 @@ public StarTreeMetadata( IndexInput meta, Integer version, Integer numberOfNodes, - LinkedHashMap dimensionFieldsToDocValuesMap, + LinkedHashMap dimensionFieldToDimensionConfigMap, List metrics, Integer segmentAggregatedDocCount, Integer starTreeDocCount, @@ -185,7 +187,7 @@ public StarTreeMetadata( this.starTreeFieldType = compositeFieldType.getName(); this.version = version; this.numberOfNodes = numberOfNodes; - this.dimensionFieldsToDocValuesMap = dimensionFieldsToDocValuesMap; + this.dimensionFieldToDimensionConfigMap = dimensionFieldToDimensionConfigMap; this.metrics = metrics; this.segmentAggregatedDocCount = segmentAggregatedDocCount; this.starTreeDocCount = starTreeDocCount; @@ -204,14 +206,24 @@ private int readDimensionsCount() throws IOException { return meta.readVInt(); } - private LinkedHashMap readStarTreeDimensions() throws IOException { + private LinkedHashMap readStarTreeDimensions() throws IOException { int dimensionCount = readDimensionsCount(); - LinkedHashMap dimensionFieldsToDocValuesMap = new LinkedHashMap<>(); + LinkedHashMap dimensionFieldToDimensionConfigMap = new LinkedHashMap<>(); for (int i = 0; i < dimensionCount; i++) { - dimensionFieldsToDocValuesMap.put(meta.readString(), getDocValuesType(meta, meta.readByte())); + if (getVersion() >= StarTreeWriter.VERSION_DIMENSION_DATA_TYPE) { + dimensionFieldToDimensionConfigMap.put( + meta.readString(), + new DimensionConfig(getDocValuesType(meta, meta.readByte()), getDimensionDataType(meta, meta.readByte())) + ); + } else { + dimensionFieldToDimensionConfigMap.put( + meta.readString(), + new DimensionConfig(getDocValuesType(meta, meta.readByte()), DimensionDataType.LONG) + ); + } } - return dimensionFieldsToDocValuesMap; + return dimensionFieldToDimensionConfigMap; } private int readMetricsCount() throws IOException { @@ -315,8 +327,8 @@ public String getStarTreeFieldType() { * * @return star-tree dimension field numbers */ - public Map getDimensionFields() { - return dimensionFieldsToDocValuesMap; + public Map getDimensionFields() { + return dimensionFieldToDimensionConfigMap; } /** @@ -393,6 +405,7 @@ public long getDataLength() { /** * Returns the version with which the star tree is stored in the segments + * * @return star-tree version */ public int getVersion() { @@ -401,6 +414,7 @@ public int getVersion() { /** * Returns the number of nodes in the star tree + * * @return number of nodes in the star tree */ public int getNumberOfNodes() { @@ -425,4 +439,15 @@ private static DocValuesType getDocValuesType(IndexInput input, byte b) throws I throw new CorruptIndexException("invalid docvalues byte: " + b, input); } } + + private static DimensionDataType getDimensionDataType(IndexInput input, byte b) throws IOException { + switch (b) { + case 0: + return DimensionDataType.LONG; + case 1: + return DimensionDataType.UNSIGNED_LONG; + default: + throw new CorruptIndexException("invalid dimensionDataType byte: " + b, input); + } + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 569692ce18893..5bf9c6174fd2f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -12,6 +12,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.mapper.CompositeMappedFieldType; @@ -131,9 +133,20 @@ private static void writeMeta( metaOut.writeVInt(starTreeField.getDimensionNames().size()); // dimensions - for (int i = 0; i < starTreeField.getDimensionNames().size(); i++) { - metaOut.writeString(starTreeField.getDimensionNames().get(i)); - metaOut.writeByte(docValuesByte(starTreeField.getDimensionDocValueTypes().get(i))); + List dimensionsOrder = starTreeField.getDimensionsOrder(); + int docDimensionIndex = 0; + for (Dimension currentDimension : dimensionsOrder) { + int numSubDimensions = currentDimension.getNumSubDimensions(); + + // Process each sub-dimension + while (numSubDimensions > 0) { + metaOut.writeString(starTreeField.getDimensionNames().get(docDimensionIndex)); + metaOut.writeByte(docValuesByte(starTreeField.getDimensionDocValueTypes().get(docDimensionIndex))); + metaOut.writeByte(dimensionDataTypeByte(currentDimension.getDimensionDataType())); + + numSubDimensions--; + docDimensionIndex++; + } } // number of metrics @@ -174,6 +187,17 @@ private static void writeMeta( } + private static byte dimensionDataTypeByte(DimensionDataType dimensionDataType) { + switch (dimensionDataType) { + case LONG: + return 0; + case UNSIGNED_LONG: + return 1; + default: + throw new AssertionError("unhandled dimensionDataType: " + dimensionDataType); + } + } + private static byte docValuesByte(DocValuesType type) { switch (type) { case NONE: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 6a13e6e789f3a..6658d53afd21a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -24,6 +24,7 @@ import org.opensearch.index.compositeindex.datacube.ReadDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; @@ -131,13 +132,15 @@ public StarTreeValues( // build dimensions List readDimensions = new ArrayList<>(); - for (String dimension : starTreeMetadata.getDimensionFields().keySet()) { + for (Map.Entry dimensionEntry : starTreeMetadata.getDimensionFields().entrySet()) { + String dimension = dimensionEntry.getKey(); readDimensions.add( new ReadDimension( dimension, readState.fieldInfos.fieldInfo( fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeMetadata.getCompositeFieldName(), dimension) - ).getDocValuesType() + ).getDocValuesType(), + dimensionEntry.getValue().getDimensionDataType() ) ); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java index 27131122d77ec..add51a8131f27 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java @@ -11,7 +11,6 @@ import org.apache.lucene.util.IntroSorter; import org.opensearch.index.compositeindex.datacube.Dimension; -import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.function.IntFunction; @@ -48,12 +47,17 @@ protected void setPivot(int i) { @Override protected int comparePivot(int j) { Long[] currentDimensions = dimensionsReader.apply(j); - for (int i = dimensionId + 1; i < dimensions.length; i++) { + int totalDimensions = dimensionsOrder.size(); + int docDimensionIndex = dimensionId + 1; + for (int i = dimensionId + 1; i < totalDimensions; i++) { Dimension dimension = dimensionsOrder.get(i); - Long dimensionValue = currentDimensions[i]; - if (!Objects.equals(dimensions[i], dimensionValue)) { - Comparator comparator = dimension.comparator(); - return comparator.compare(dimensions[i], dimensionValue); + int subDimensionsToProcess = dimension.getNumSubDimensions(); + while (subDimensionsToProcess > 0) { + if (!Objects.equals(dimensions[docDimensionIndex], currentDimensions[docDimensionIndex])) { + return dimension.comparator().compare(dimensions[docDimensionIndex], currentDimensions[docDimensionIndex]); + } + docDimensionIndex++; + subDimensionsToProcess--; } } return 0; diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index 93dcf3b32da07..b2572af042b9c 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -90,7 +90,7 @@ public void testStarTreeDocValues() throws IOException { // Segment documents /** - * sndv dv unsignedLongDimension field + * unsignedLongDimension sndv dv field * [10, 1, 1, -1] * [10, 1, 1, -1] * [-20, 2, 2, -2] diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java index 03dd681303070..99ade73d242ae 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java @@ -35,6 +35,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; @@ -447,7 +448,7 @@ public static void validateStarTreeFileFormats( public static SegmentReadState getReadState( int numDocs, - Map dimensionFields, + Map dimensionFields, List metrics, StarTreeField compositeField, SegmentWriteState writeState, @@ -470,7 +471,7 @@ public static SegmentReadState getReadState( false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - dimensionFields.get(dimension), + dimensionFields.get(dimension).getDocValuesType(), -1, Collections.emptyMap(), 0, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java index bc2f5cd0c3fd3..723fb23865170 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java @@ -30,12 +30,14 @@ import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; @@ -717,11 +719,11 @@ public void test_build_multipleStarTrees() throws IOException { metaOut.close(); dataOut.close(); - LinkedHashMap fieldsMap = new LinkedHashMap<>(); - fieldsMap.put("field1", DocValuesType.SORTED_NUMERIC); - fieldsMap.put("field3", DocValuesType.SORTED_NUMERIC); - fieldsMap.put("field5", DocValuesType.SORTED_NUMERIC); - fieldsMap.put("field8", DocValuesType.SORTED_NUMERIC); + LinkedHashMap fieldsMap = new LinkedHashMap<>(); + fieldsMap.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap.put("field5", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap.put("field8", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "test", @@ -746,10 +748,10 @@ public void test_build_multipleStarTrees() throws IOException { 330 ); - LinkedHashMap fieldsMap1 = new LinkedHashMap<>(); - fieldsMap1.put("fieldC", DocValuesType.SORTED_NUMERIC); - fieldsMap1.put("fieldB", DocValuesType.SORTED_NUMERIC); - fieldsMap1.put("fieldL", DocValuesType.SORTED_NUMERIC); + LinkedHashMap fieldsMap1 = new LinkedHashMap<>(); + fieldsMap1.put("fieldC", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap1.put("fieldB", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + fieldsMap1.put("fieldL", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata2 = new StarTreeMetadata( "test", @@ -768,7 +770,7 @@ public void test_build_multipleStarTrees() throws IOException { 1287 ); - LinkedHashMap totalDimensionFields = new LinkedHashMap<>(starTreeMetadata.getDimensionFields()); + LinkedHashMap totalDimensionFields = new LinkedHashMap<>(starTreeMetadata.getDimensionFields()); totalDimensionFields.putAll(starTreeMetadata2.getDimensionFields()); List metrics = new ArrayList<>(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index b4c0c680373ee..45ce848425d84 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -20,6 +20,7 @@ import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.KeywordDimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -28,6 +29,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; @@ -131,9 +133,9 @@ public void testFlushFlow() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_NUMERIC); - docValues.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, @@ -231,9 +233,9 @@ public void testFlushFlowDimsReverse() throws IOException { dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_NUMERIC); - docValues.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, @@ -344,9 +346,9 @@ public void testFlushFlowWithUnsignedLongDimensions() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_NUMERIC); - docValues.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, @@ -447,9 +449,9 @@ public void testFlushFlowBuild() throws IOException { dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 100, 1, 6699); validateStarTreeFileFormats( @@ -604,9 +606,9 @@ public void testFlushFlowForKeywords() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap docValues = new LinkedHashMap<>(); - docValues.put("field1", DocValuesType.SORTED_SET); - docValues.put("field3", DocValuesType.SORTED_SET); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", new DimensionConfig(DocValuesType.SORTED_SET, DimensionDataType.LONG)); + docValues.put("field3", new DimensionConfig(DocValuesType.SORTED_SET, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index 90b283f235168..01bf698f2452e 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -18,8 +18,8 @@ import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.CompositeIndexConstants; -import org.opensearch.index.compositeindex.datacube.ComparatorType; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.DimensionDataType; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; @@ -27,6 +27,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; @@ -286,10 +287,10 @@ public void testMergeFlowForUnsignedLong() throws IOException { metricsWithFieldValueCount.add(i); } - Dimension d1 = new ReadDimension("field1", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); - Dimension d2 = new ReadDimension("field3", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); - Dimension d3 = new ReadDimension("field5", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); - Dimension d4 = new ReadDimension("field8", DocValuesType.SORTED_NUMERIC, ComparatorType.UNSIGNED_LONG); + Dimension d1 = new ReadDimension("field1", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + Dimension d2 = new ReadDimension("field3", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + Dimension d3 = new ReadDimension("field5", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); + Dimension d4 = new ReadDimension("field8", DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG); Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); @@ -538,9 +539,9 @@ public void testMergeFlowWithSum() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -625,9 +626,9 @@ public void testMergeFlowForUnsignedLongWithSum() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.UNSIGNED_LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -700,9 +701,9 @@ public void testMergeFlowWithCount() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -849,9 +850,9 @@ public void testMergeFlowWithMissingDocs() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( @@ -939,9 +940,9 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 231); validateStarTreeFileFormats( @@ -1032,9 +1033,9 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 7, 1000, 231); validateStarTreeFileFormats( @@ -1121,9 +1122,9 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( @@ -1208,9 +1209,9 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( @@ -1283,9 +1284,9 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( @@ -1684,9 +1685,9 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - LinkedHashMap map = new LinkedHashMap<>(); - map.put("field1", DocValuesType.SORTED_NUMERIC); - map.put("field3", DocValuesType.SORTED_NUMERIC); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); + map.put("field3", new DimensionConfig(DocValuesType.SORTED_NUMERIC, DimensionDataType.LONG)); StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 9, 1000, 330); validateStarTreeFileFormats( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java index 0f873cdc55238..59af8dd2c99a8 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java @@ -41,6 +41,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.DimensionConfig; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; @@ -248,7 +249,7 @@ SegmentWriteState getWriteState(int numDocs, byte[] id) { return BuilderTestsUtils.getWriteState(numDocs, id, fieldsInfo, directory); } - SegmentReadState getReadState(int numDocs, Map dimensionFields, List metrics) { + SegmentReadState getReadState(int numDocs, Map dimensionFields, List metrics) { return BuilderTestsUtils.getReadState(numDocs, dimensionFields, metrics, compositeField, writeState, directory); } @@ -256,11 +257,11 @@ protected Map getAttributes(int numSegmentDocs) { return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); } - protected LinkedHashMap getStarTreeDimensionNames(List dimensionsOrder) { - LinkedHashMap dimensionNames = new LinkedHashMap<>(); + protected LinkedHashMap getStarTreeDimensionNames(List dimensionsOrder) { + LinkedHashMap dimensionNames = new LinkedHashMap<>(); for (Dimension dimension : dimensionsOrder) { for (String dimensionName : dimension.getSubDimensionNames()) { - dimensionNames.put(dimensionName, dimension.getDocValuesType()); + dimensionNames.put(dimensionName, new DimensionConfig(dimension.getDocValuesType(), dimension.getDimensionDataType())); } } return dimensionNames; @@ -395,7 +396,7 @@ protected long getLongFromDouble(double value) { } protected StarTreeMetadata getStarTreeMetadata( - LinkedHashMap fields, + LinkedHashMap fields, int segmentAggregatedDocCount, int maxLeafDocs, int dataLength diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index cc91d69be97c1..a862d082b5ec8 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -42,7 +42,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.UUID; @@ -182,11 +181,10 @@ public void test_starTreeMetadata() throws IOException { assertEquals(starTreeMetadata.getNumberOfNodes(), numberOfNodes); assertNotNull(starTreeMetadata); - assertEquals(dimensionsOrder.size(), starTreeMetadata.dimensionFieldsToDocValuesMap.size()); - int k = 0; - for (Map.Entry entry : starTreeMetadata.dimensionFieldsToDocValuesMap.entrySet()) { - assertEquals(dimensionsOrder.get(k).getField(), entry.getKey()); - k++; + assertEquals(dimensionsOrder.size(), starTreeMetadata.dimensionFieldToDimensionConfigMap.size()); + int index = 0; + for (String key : starTreeMetadata.dimensionFieldToDimensionConfigMap.keySet()) { + assertEquals(dimensionsOrder.get(index++).getField(), key); } assertEquals(starTreeField.getMetrics().size(), starTreeMetadata.getMetrics().size()); From 87742cc9153d97744d6b619ddb491efd6b22b653 Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Tue, 21 Jan 2025 10:55:31 +0530 Subject: [PATCH 04/10] added tests --- .../index/mapper/StarTreeMapperIT.java | 10 +++ .../datacube/startree/StarTreeTestUtils.java | 5 ++ .../node/FixedLengthStarTreeNodeTests.java | 65 +++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 1d01f717aad1f..b657350ed6cef 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -76,6 +76,9 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .startObject() .field("name", "keyword_dv") .endObject() + .startObject() + .field("name", "unsignedLongDimension") // UnsignedLongDimension + .endObject() .endArray() .startArray("metrics") .startObject() @@ -117,6 +120,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "wildcard") .field("doc_values", false) .endObject() + .startObject("unsignedLongDimension") + .field("type", "unsigned_long") + .field("doc_values", true) + .endObject() .endObject() .endObject(); } catch (IOException e) { @@ -605,8 +612,11 @@ public void testValidCompositeIndex() { for (int i = 0; i < dateDim.getSortedCalendarIntervals().size(); i++) { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); } + assertEquals(4, starTreeFieldType.getDimensions().size()); assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("keyword_dv", starTreeFieldType.getDimensions().get(2).getField()); + assertEquals("unsignedLongDimension", starTreeFieldType.getDimensions().get(3).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index d2b663ed7a6fc..5666a3ade5513 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -209,6 +209,11 @@ public static void validateFileFormats( int childCount = 0; boolean childStarNodeAsserted = false; boolean nodeWithMinusOneValueFound = false; + /* + Since NULL nodes have a dimension value of -1, we need to track whether we have encountered any + default nodes with this dimension value. We will perform the assertNull() check only if we have not + yet found a default node with a dimension value of -1. + */ while (expectedChildrenIterator.hasNext()) { StarTreeNode child = expectedChildrenIterator.next(); InMemoryTreeNode resultChildNode = null; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java index 3f2c6c89f45dd..e8052f8099a9c 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; @@ -231,6 +232,70 @@ public void testOnlyRootNodePresent() throws IOException { directory.close(); } + public void testGetChildForUnsignedLongDimensionValue() throws IOException { + + Directory directory = newFSDirectory(createTempDir()); + IndexOutput dataOut = directory.createOutput("star-tree-data-2", IOContext.DEFAULT); + StarTreeWriter starTreeWriter = new StarTreeWriter(); + + InMemoryTreeNode node = new InMemoryTreeNode(0, randomInt(), randomInt(), randomFrom((byte) 0, (byte) -1, (byte) 1), -1); + node.setChildDimensionId(1); + node.setAggregatedDocId(randomInt()); + + InMemoryTreeNode starChild = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) -1, -1); + starChild.setChildDimensionId(-1); + starChild.setAggregatedDocId(randomInt()); + node.addChildNode(starChild, (long) ALL); + + InMemoryTreeNode childWithMinus1 = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) 0, -1); + childWithMinus1.setChildDimensionId(-1); + childWithMinus1.setAggregatedDocId(randomInt()); + + for (int i = 1; i < randomIntBetween(2, 5); i++) { + InMemoryTreeNode child = new InMemoryTreeNode( + node.getDimensionId() + 1, + randomInt(), + randomInt(), + (byte) 0, + node.getDimensionValue() + i + ); + child.setChildDimensionId(-1); + child.setAggregatedDocId(randomInt()); + node.addChildNode(child, child.getDimensionValue()); + } + + node.addChildNode(childWithMinus1, -1L); + + InMemoryTreeNode nullChild = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) 1, -1); + nullChild.setChildDimensionId(-1); + nullChild.setAggregatedDocId(randomInt()); + node.addChildNode(nullChild, null); + + long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, node, 2 + node.getChildren().size(), "star-tree"); + + // asserting on the actual length of the star tree data file + assertEquals(starTreeDataLength, 33L * node.getChildren().size() + 2 * 33); + dataOut.close(); + + IndexInput dataIn = directory.openInput("star-tree-data-2", IOContext.READONCE); + StarTreeMetadata starTreeMetadata = mock(StarTreeMetadata.class); + when(starTreeMetadata.getDataLength()).thenReturn(starTreeDataLength); + when(starTreeMetadata.getDataStartFilePointer()).thenReturn(0L); + + FixedLengthStarTreeNode starTreeNode = (FixedLengthStarTreeNode) StarTreeFactory.createStarTree(dataIn, starTreeMetadata); + + long dimensionValue = -1; + FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue( + dimensionValue, + new UnsignedLongDimension("field") + ); + assertNotNull(childNode); + assertEquals(dimensionValue, childNode.getDimensionValue()); + + dataIn.close(); + directory.close(); + } + public void tearDown() throws Exception { super.tearDown(); dataIn.close(); From 7dfaeedfa6bc130e0d43f21fb48639711f033d9f Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Tue, 21 Jan 2025 10:55:31 +0530 Subject: [PATCH 05/10] added tests --- .../compositeindex/datacube/startree/node/StarTreeNode.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index bebf7d476d43b..84d3436dc6cfb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -90,13 +90,13 @@ public interface StarTreeNode { * *

The node type can be one of the following: *

    - *
  • Star Node: Represented by the value -2. - *
  • Null Node: Represented by the value -1. + *
  • Star Node: Represented by the value -1. + *
  • Null Node: Represented by the value 1. *
  • Default Node: Represented by the value 0. *
* @see StarTreeNodeType * - * @return The type of the current node, represented by the corresponding integer value (-2, -1, or 0). + * @return The type of the current node, represented by the corresponding integer value (-1, 1, 0). * @throws IOException if an I/O error occurs while reading the node type */ byte getStarTreeNodeType() throws IOException; From fa447c850bb975f63b05c2d1f919eaea5bf30e08 Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Wed, 22 Jan 2025 11:51:48 +0530 Subject: [PATCH 06/10] add dimesnionComparators list in star tree sorting logic --- .../startree/builder/BaseStarTreeBuilder.java | 4 + .../builder/OffHeapStarTreeBuilder.java | 4 +- .../builder/OnHeapStarTreeBuilder.java | 21 +-- .../utils/StarTreeDocumentsSorter.java | 18 +- .../utils/StarTreeDocumentsSorterTests.java | 164 ++++++++++-------- 5 files changed, 112 insertions(+), 99 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index cf36f2d7d4126..de5800eec19f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -55,6 +55,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -112,6 +113,8 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { // This should be true for merge flows protected boolean isMerge = false; + protected final List> dimensionComparators = new ArrayList<>(); + /** * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. * @@ -136,6 +139,7 @@ protected BaseStarTreeBuilder( int numDims = 0; for (Dimension dim : starTreeField.getDimensionsOrder()) { numDims += dim.getNumSubDimensions(); + dimensionComparators.addAll(Collections.nCopies(dim.getNumSubDimensions(), dim.comparator())); dimensionsSplitOrder.add(dim); } this.numDimensions = numDims; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 03b462a13ce1a..da48559461c49 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -235,7 +235,7 @@ private Iterator sortAndReduceDocuments(int[] sortedDocIds, in } catch (IOException e) { throw new UncheckedIOException(e); } - }, starTreeDocumentFileManager.starTreeField.getDimensionsOrder()); + }, dimensionComparators); } catch (UncheckedIOException ex) { // Unwrap UncheckedIOException and throw as IOException if (ex.getCause() != null) { @@ -335,7 +335,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } catch (IOException e) { throw new RuntimeException(e); } - }, starTreeDocumentFileManager.starTreeField.getDimensionsOrder()); + }, dimensionComparators); // Create an iterator for aggregated documents return new Iterator() { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index e4dc561d102a1..a590b2b69cbc8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -13,7 +13,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -161,7 +160,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments, boolean isMerge) { // sort all the documents - sortStarTreeDocumentsFromDimensionId(starTreeDocuments, 0); + sortStarTreeDocumentsFromDimensionId(starTreeDocuments, -1); // merge the documents return mergeStarTreeDocuments(starTreeDocuments, isMerge); @@ -222,7 +221,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } // sort star tree documents from given dimension id (as previous dimension ids have already been processed) - sortStarTreeDocumentsFromDimensionId(starTreeDocuments, dimensionId + 1); + sortStarTreeDocumentsFromDimensionId(starTreeDocuments, dimensionId); return new Iterator() { boolean hasNext = true; @@ -271,19 +270,9 @@ public StarTreeDocument next() { */ private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) { Arrays.sort(starTreeDocuments, (doc1, doc2) -> { - List dimensionsOrder = starTreeField.getDimensionsOrder(); - int totalDimensions = dimensionsOrder.size(); - - int docDimensionIndex = dimensionId; - for (int i = dimensionId; i < totalDimensions; i++) { - Dimension dimension = dimensionsOrder.get(i); - int subDimensionsToProcess = dimension.getNumSubDimensions(); - while (subDimensionsToProcess > 0) { - if (!Objects.equals(doc1.dimensions[docDimensionIndex], doc2.dimensions[docDimensionIndex])) { - return dimension.comparator().compare(doc1.dimensions[docDimensionIndex], doc2.dimensions[docDimensionIndex]); - } - docDimensionIndex++; - subDimensionsToProcess--; + for (int i = dimensionId + 1; i < numDimensions; i++) { + if (!Objects.equals(doc1.dimensions[i], doc2.dimensions[i])) { + return dimensionComparators.get(i).compare(doc1.dimensions[i], doc2.dimensions[i]); } } return 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java index add51a8131f27..09a653f13cf28 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorter.java @@ -9,8 +9,8 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.apache.lucene.util.IntroSorter; -import org.opensearch.index.compositeindex.datacube.Dimension; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.function.IntFunction; @@ -27,7 +27,7 @@ public static void sort( final int dimensionId, final int numDocs, final IntFunction dimensionsReader, - final List dimensionsOrder + final List> dimensionComparators ) { new IntroSorter() { private Long[] dimensions; @@ -47,17 +47,9 @@ protected void setPivot(int i) { @Override protected int comparePivot(int j) { Long[] currentDimensions = dimensionsReader.apply(j); - int totalDimensions = dimensionsOrder.size(); - int docDimensionIndex = dimensionId + 1; - for (int i = dimensionId + 1; i < totalDimensions; i++) { - Dimension dimension = dimensionsOrder.get(i); - int subDimensionsToProcess = dimension.getNumSubDimensions(); - while (subDimensionsToProcess > 0) { - if (!Objects.equals(dimensions[docDimensionIndex], currentDimensions[docDimensionIndex])) { - return dimension.comparator().compare(dimensions[docDimensionIndex], currentDimensions[docDimensionIndex]); - } - docDimensionIndex++; - subDimensionsToProcess--; + for (int i = dimensionId + 1; i < dimensions.length; i++) { + if (!Objects.equals(dimensions[i], currentDimensions[i])) { + return dimensionComparators.get(i).compare(dimensions[i], currentDimensions[i]); } } return 0; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java index 6b17bab5d8672..ce1d36024524b 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentsSorterTests.java @@ -9,19 +9,25 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.opensearch.common.Randomness; -import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.common.Rounding; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; /** * Tests for {@link StarTreeDocumentsSorter}. @@ -29,77 +35,114 @@ public class StarTreeDocumentsSorterTests extends OpenSearchTestCase { private Map testData; - private List dimensionsOrder; + private List> comparatorList; @Before public void setUp() throws Exception { super.setUp(); testData = new HashMap<>(); + comparatorList = new ArrayList<>(); - // 10 documents with 5 dimensions each - testData.put(0, new Long[] { null, 150L, 100L, 300L, null }); - testData.put(1, new Long[] { 1L, null, -9223372036854775807L, 200L, 300L }); - testData.put(2, new Long[] { 2L, -100L, -15L, 250L, null }); - testData.put(3, new Long[] { 2L, -100L, -10L, 210L, -9223372036854775807L }); - testData.put(4, new Long[] { 1L, 120L, null, null, 305L }); - testData.put(5, new Long[] { 2L, 150L, -5L, 200L, 295L }); - testData.put(6, new Long[] { 3L, 105L, null, -200L, -315L }); - testData.put(7, new Long[] { 1L, 120L, -10L, 205L, 310L }); - testData.put(8, new Long[] { null, -100L, 9223372036854775807L, 200L, -300L }); - testData.put(9, new Long[] { 2L, null, -10L, 210L, 325L }); - - dimensionsOrder = Arrays.asList( - new NumericDimension("dim1"), // Long - new UnsignedLongDimension("dim2"), // Unsigned Long - new NumericDimension("dim3"), // Long - new UnsignedLongDimension("dim4"), // Unsigned Long - new NumericDimension("dim5") // Long + List intervals = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY) ); + DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Long[] date_dims = new Long[4]; + Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { date_dims[dimIndex.getAndIncrement()] = value; }); + + // 10 documents with 6 dimensions each + testData.put(0, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], null, 150L, 100L, 300L, null }); + testData.put(1, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 1L, null, -9223372036854775807L, 200L, 300L }); + testData.put(2, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, -100L, -15L, 250L, null }); + testData.put( + 3, + new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, -100L, -10L, 210L, -9223372036854775807L } + ); + testData.put(4, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 1L, 120L, null, null, 305L }); + testData.put(5, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, 150L, -5L, 200L, 295L }); + testData.put(6, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 3L, 105L, null, -200L, -315L }); + testData.put(7, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 1L, 120L, -10L, 205L, 310L }); + testData.put( + 8, + new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], null, -100L, 9223372036854775807L, 200L, -300L } + ); + testData.put(9, new Long[] { date_dims[0], date_dims[1], date_dims[2], date_dims[3], 2L, null, -10L, 210L, 325L }); + + comparatorList.addAll(Collections.nCopies(4, dateDimension.comparator())); + comparatorList.add(new NumericDimension("dim1").comparator()); + comparatorList.add(new UnsignedLongDimension("dim2").comparator()); + comparatorList.add(new NumericDimension("dim3").comparator()); + comparatorList.add(new UnsignedLongDimension("dim4").comparator()); + comparatorList.add(new NumericDimension("dim5").comparator()); + } - public void testSortDocumentsOffHeap_FirstDimension() { + public void testSortDocumentsOffHeap_StartFromFirstDimension() { int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = -1; int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); } - public void testSortDocumentsOffHeap_SecondDimension() { + public void testSortDocumentsOffHeap_StartFromSecondDimension() { int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = 0; int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); - assertArrayEquals(new int[] { 6, 7, 4, 5, 0, 2, 3, 8, 1, 9 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); } - public void testSortDocumentsOffHeap_ThirdDimension() { + public void testSortDocumentsOffHeap_StartFromThirdDimension() { int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = 1; int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); - assertArrayEquals(new int[] { 1, 2, 7, 3, 9, 5, 0, 8, 6, 4 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); } - public void testSortDocumentsOffHeap_FourthDimension() { + public void testSortDocumentsOffHeap_StartFromFourthDimension() { int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = 2; int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); - assertArrayEquals(new int[] { 8, 5, 1, 7, 3, 9, 2, 0, 6, 4 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); } - public void testSortDocumentsOffHeap_FifthDimension() { + public void testSortDocumentsOffHeap_StartFromFifthDimension() { int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; int dimensionId = 3; int numDocs = 10; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); - assertArrayEquals(new int[] { 3, 6, 8, 5, 1, 4, 7, 9, 0, 2 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 7, 4, 1, 5, 2, 3, 9, 6, 0, 8 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_StartFromSixthDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 4; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 6, 7, 4, 5, 0, 2, 3, 8, 1, 9 }, sortedDocIds); + } + + public void testSortDocumentsOffHeap_StartFromSeventhDimension() { + int[] sortedDocIds = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + int dimensionId = 5; + int numDocs = 10; + + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 1, 2, 7, 3, 9, 5, 0, 8, 6, 4 }, sortedDocIds); } public void testSortDocumentsOffHeap_SingleElement() { @@ -107,7 +150,7 @@ public void testSortDocumentsOffHeap_SingleElement() { int dimensionId = -1; int numDocs = 1; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); assertArrayEquals(new int[] { 0 }, sortedDocIds); } @@ -116,21 +159,21 @@ public void testSortDocumentsOffHeap_EmptyArray() { int dimensionId = -1; int numDocs = 0; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); assertArrayEquals(new int[] {}, sortedDocIds); } public void testSortDocumentsOffHeap_AllNulls() { Map testData = new HashMap<>(); - testData.put(0, new Long[] { null, null, null, null, null }); - testData.put(1, new Long[] { null, null, null, null, null }); - testData.put(2, new Long[] { null, null, null, null, null }); + testData.put(0, new Long[] { null, null, null, null, null, null, null, null, null }); + testData.put(1, new Long[] { null, null, null, null, null, null, null, null, null }); + testData.put(2, new Long[] { null, null, null, null, null, null, null, null, null }); int[] sortedDocIds = { 0, 1, 2 }; int dimensionId = -1; int numDocs = 3; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); // The order should remain unchanged as all elements are equal (null) assertArrayEquals(new int[] { 0, 1, 2 }, sortedDocIds); @@ -138,18 +181,18 @@ public void testSortDocumentsOffHeap_AllNulls() { public void testSortDocumentsOffHeap_Negatives() { Map testData = new HashMap<>(); - testData.put(0, new Long[] { -10L, 0L, null, 0L, -5L }); - testData.put(1, new Long[] { -9L, 0L, null, 0L, -10L }); - testData.put(2, new Long[] { -9L, 0L, null, 0L, 15L }); - testData.put(3, new Long[] { -7L, 0L, null, 0L, -20L }); - testData.put(4, new Long[] { -15L, 0L, null, 0L, -25L }); + testData.put(0, new Long[] { -1L, -2L, -3L, -4L, -10L, 0L, null, 0L, -5L }); + testData.put(1, new Long[] { -5L, -2L, -3L, -4L, -9L, 0L, null, 0L, -10L }); + testData.put(2, new Long[] { -5L, -3L, -3L, -4L, -9L, 0L, null, 0L, 15L }); + testData.put(3, new Long[] { -9L, -2L, -3L, -4L, -7L, 0L, null, 0L, -20L }); + testData.put(4, new Long[] { -8L, -2L, -3L, -4L, -15L, 0L, null, 0L, -25L }); int[] sortedDocIds = { 0, 1, 2, 3, 4 }; int dimensionId = -1; int numDocs = 5; - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); - assertArrayEquals(new int[] { 4, 0, 1, 2, 3 }, sortedDocIds); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); + assertArrayEquals(new int[] { 3, 4, 2, 1, 0 }, sortedDocIds); } public void testTheRandomSort() { @@ -187,17 +230,16 @@ private void testRandomizedSort() { // sort dimensionId + 1 to numDimensions // for example to start from dimension in 0th index, we need to pass -1 to sort method int dimensionId = random.nextInt(numDimensions) - 1; + List> comparatorList = new ArrayList<>(); - List dimensionsOrder = new ArrayList<>(); for (int i = 0; i < numDimensions; i++) { Boolean isUnsignedLong = random.nextBoolean(); - - if (!isUnsignedLong) dimensionsOrder.add(new NumericDimension("fieldName")); - else dimensionsOrder.add(new UnsignedLongDimension("fieldName")); + if (!isUnsignedLong) comparatorList.add(new NumericDimension("fieldName").comparator()); + else comparatorList.add(new UnsignedLongDimension("fieldName").comparator()); } // Sort using StarTreeDocumentsSorter - StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), dimensionsOrder); + StarTreeDocumentsSorter.sort(sortedDocIds, dimensionId, numDocs, i -> testData.get(sortedDocIds[i]), comparatorList); // Verify the sorting for (int i = 1; i < numDocs; i++) { @@ -205,9 +247,7 @@ private void testRandomizedSort() { Long[] curr = testData.get(sortedDocIds[i]); boolean isCorrectOrder = true; for (int j = dimensionId + 1; j < numDimensions; j++) { - int comparison = -1; - if (dimensionsOrder.get(j) instanceof UnsignedLongDimension) comparison = compareLongs(prev[j], curr[j], true); - else comparison = compareLongs(prev[j], curr[j], false); + int comparison = comparatorList.get(j).compare(prev[j], curr[j]); if (comparison < 0) { break; } else if (comparison > 0) { @@ -227,16 +267,4 @@ private void testRandomizedSort() { } } - private int compareLongs(Long a, Long b, Boolean isUnsignedLong) { - if (!Objects.equals(a, b)) { - if (a == null) { - return 1; - } else if (b == null) { - return -1; - } else { - return isUnsignedLong ? Long.compareUnsigned(a, b) : Long.compare(a, b); - } - } - return 0; - } } From 1394c39036e56d142b5b4c8a6281ad86a27c9283 Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Mon, 27 Jan 2025 13:07:35 +0530 Subject: [PATCH 07/10] address comments - change Collection.ncopies to a for loop --- .../index/compositeindex/datacube/NumericDimension.java | 1 - .../datacube/startree/builder/BaseStarTreeBuilder.java | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index 1cd46327feb04..fe9e3d17c0047 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -77,5 +77,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } - } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index de5800eec19f8..935c490b5a4dc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -139,7 +139,9 @@ protected BaseStarTreeBuilder( int numDims = 0; for (Dimension dim : starTreeField.getDimensionsOrder()) { numDims += dim.getNumSubDimensions(); - dimensionComparators.addAll(Collections.nCopies(dim.getNumSubDimensions(), dim.comparator())); + for (int i = 0; i < dim.getNumSubDimensions(); i++) { + dimensionComparators.add(dim.comparator()); + } dimensionsSplitOrder.add(dim); } this.numDimensions = numDims; From 25ab68803ff2180ecdbf3b7b3a317380b07eb3f3 Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Mon, 27 Jan 2025 17:15:35 +0530 Subject: [PATCH 08/10] refractor and add java docs --- .../fileformats/node/FixedLengthStarTreeNode.java | 1 + .../datacube/startree/node/StarTreeNode.java | 1 + .../org/opensearch/index/mapper/NumberFieldMapper.java | 8 +++----- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index 8f67b611f08ca..92dc8c055bad6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -239,6 +239,7 @@ private static FixedLengthStarTreeNode matchStarTreeNodeTypeOrNull(FixedLengthSt * Performs a binary search to find a child node with the given dimension value. * * @param dimensionValue The dimension value to search for + * @param dimension the parent dimension node * @return The child node if found, null otherwise * @throws IOException If there's an error reading from the input */ diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index 84d3436dc6cfb..a514b00ae8a89 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -105,6 +105,7 @@ public interface StarTreeNode { * Returns the child node for the given dimension value in the star-tree. * * @param dimensionValue the dimension value + * @param dimension the parent dimension node * @return the child node for the given dimension value or null if child is not present * @throws IOException if an I/O error occurs while retrieving the child node */ diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 713e891629e02..feeb60cd92195 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -177,11 +177,9 @@ public NumberFieldMapper build(BuilderContext context) { @Override public Optional getSupportedDataCubeDimensionType() { - return isUnsignedLong() ? Optional.of(DimensionType.UNSIGNED_LONG) : Optional.of(DimensionType.NUMERIC); - } - - private Boolean isUnsignedLong() { - return type.numericType.equals(NumericType.UNSIGNED_LONG); + return type.numericType.equals(NumericType.UNSIGNED_LONG) + ? Optional.of(DimensionType.UNSIGNED_LONG) + : Optional.of(DimensionType.NUMERIC); } @Override From 51eca0a1757afe33a9de7c92ad1d99ce0cdad2ef Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Mon, 27 Jan 2025 20:48:40 +0530 Subject: [PATCH 09/10] remove query changes for unsigned-long --- .../node/FixedLengthStarTreeNode.java | 14 ++-- .../datacube/startree/node/StarTreeNode.java | 3 +- .../search/startree/StarTreeFilter.java | 2 +- .../datacube/startree/StarTreeTestUtils.java | 4 +- .../StarTreeBuilderFlushFlowTests.java | 12 +-- .../StarTreeBuilderMergeFlowTests.java | 12 +-- .../data/StarTreeFileFormatsTests.java | 5 +- .../node/FixedLengthStarTreeNodeTests.java | 77 +------------------ 8 files changed, 28 insertions(+), 101 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index 92dc8c055bad6..1a75ebf685b24 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -194,7 +194,7 @@ public StarTreeNode getChildStarNode() throws IOException { } @Override - public StarTreeNode getChildForDimensionValue(Long dimensionValue, Dimension dimension) throws IOException { + public StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException { // there will be no children for leaf nodes if (isLeaf()) { return null; @@ -202,7 +202,7 @@ public StarTreeNode getChildForDimensionValue(Long dimensionValue, Dimension dim StarTreeNode resultStarTreeNode = null; if (null != dimensionValue) { - resultStarTreeNode = binarySearchChild(dimensionValue, dimension); + resultStarTreeNode = binarySearchChild(dimensionValue); } return resultStarTreeNode; } @@ -239,11 +239,10 @@ private static FixedLengthStarTreeNode matchStarTreeNodeTypeOrNull(FixedLengthSt * Performs a binary search to find a child node with the given dimension value. * * @param dimensionValue The dimension value to search for - * @param dimension the parent dimension node * @return The child node if found, null otherwise * @throws IOException If there's an error reading from the input */ - private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, Dimension dimension) throws IOException { + private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IOException { int low = firstChildId; @@ -258,15 +257,14 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, Dimension high--; } - Comparator comparator = dimension.comparator(); while (low <= high) { int mid = low + (high - low) / 2; FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid); long midDimensionValue = midNode.getDimensionValue(); - int compare = comparator.compare(midDimensionValue, dimensionValue); - if (compare == 0) { + + if (midDimensionValue == dimensionValue) { return midNode; - } else if (compare < 0) { + } else if (midDimensionValue < dimensionValue) { low = mid + 1; } else { high = mid - 1; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index a514b00ae8a89..c3ae472ab3a58 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -105,11 +105,10 @@ public interface StarTreeNode { * Returns the child node for the given dimension value in the star-tree. * * @param dimensionValue the dimension value - * @param dimension the parent dimension node * @return the child node for the given dimension value or null if child is not present * @throws IOException if an I/O error occurs while retrieving the child node */ - StarTreeNode getChildForDimensionValue(Long dimensionValue, Dimension dimension) throws IOException; + StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException; /** * Returns the child star node for a node in the star-tree. diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java index 0e062db0ddcf8..261430f542d02 100644 --- a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java @@ -167,7 +167,7 @@ private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Ma if (remainingPredicateColumns.contains(childDimension)) { long queryValue = queryMap.get(childDimension); // Get the query value directly from the map - StarTreeNode matchingChild = starTreeNode.getChildForDimensionValue(queryValue, dimensionsOrder.get(dimensionId + 1)); + StarTreeNode matchingChild = starTreeNode.getChildForDimensionValue(queryValue); if (matchingChild != null) { queue.add(matchingChild); foundLeafNode |= matchingChild.isLeaf(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 5666a3ade5513..5ede294c98754 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -229,13 +229,13 @@ default nodes with this dimension value. We will perform the assertNull() check assertNotNull(child); assertNotNull(resultChildNode); if (child.getStarTreeNodeType() == StarTreeNodeType.NULL.getValue() && !nodeWithMinusOneValueFound) { - assertNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue(), dimensionsOrder.get(dimensionId))); + assertNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); } else { if (child.getDimensionValue() == -1L) { nodeWithMinusOneValueFound = true; } assertNotNull( - starTreeNode.getChildForDimensionValue(child.getDimensionValue(), dimensionsOrder.get(dimensionId)) + starTreeNode.getChildForDimensionValue(child.getDimensionValue()) ); } assertStarTreeNode(child, resultChildNode); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index 8df160ca4c544..f4b6e5aa04b00 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -367,12 +367,12 @@ public void testFlushFlowWithUnsignedLongDimensions() throws IOException { 264 ); - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); +// validateStarTreeFileFormats( +// builder.getRootNode(), +// builder.getStarTreeDocuments().size(), +// starTreeMetadata, +// builder.getStarTreeDocuments() +// ); } public void testFlushFlowBuild() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index 37581a74823b6..0d61ba9267625 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -423,12 +423,12 @@ public void testMergeFlowForUnsignedLong() throws IOException { 132165 ); - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); +// validateStarTreeFileFormats( +// builder.getRootNode(), +// builder.getStarTreeDocuments().size(), +// starTreeMetadata, +// builder.getStarTreeDocuments() +// ); } public void testMergeFlow_randomNumberTypes() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java index 7328562d03dad..68ecf3ceee642 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java @@ -81,7 +81,7 @@ public void test_StarTreeNode() throws IOException { StarTreeNode child = childrenIterator.next(); if (child.getStarTreeNodeType() == StarTreeNodeType.DEFAULT.getValue()) { assertStarTreeNode( - starTreeNode.getChildForDimensionValue(child.getDimensionValue(), new NumericDimension(("field"))), + starTreeNode.getChildForDimensionValue(child.getDimensionValue()), inMemoryTreeNodeMap.get(child.getDimensionValue()) ); assertNull(starTreeNode.getChildStarNode()); @@ -123,8 +123,7 @@ public void test_starTreeSearch() throws IOException { for (int i = 0; i < maxLevels - 1; i++) { InMemoryTreeNode randomChildNode = randomFrom(inMemoryTreeNode.getChildren().values()); StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue( - randomChildNode.getDimensionValue(), - new NumericDimension("field") + randomChildNode.getDimensionValue() ); assertNotNull(randomStarTreeChildNode); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java index e8052f8099a9c..08815d5ef55f5 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java @@ -12,8 +12,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.compositeindex.datacube.NumericDimension; -import org.opensearch.index.compositeindex.datacube.UnsignedLongDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; @@ -143,10 +141,7 @@ public void testGetStarTreeNodeType() throws IOException { public void testGetChildForDimensionValue() throws IOException { long dimensionValue = randomIntBetween(-1, node.getChildren().size() - 3); - FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue( - dimensionValue, - new NumericDimension("field") - ); + FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(dimensionValue); assertNotNull(childNode); assertEquals(dimensionValue, childNode.getDimensionValue()); } @@ -190,13 +185,13 @@ public void testGetChildForStarNode() throws IOException { } public void testGetChildForNullNode() throws IOException { - FixedLengthStarTreeNode nullNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(null, null); + FixedLengthStarTreeNode nullNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(null); assertNull(nullNode); } public void testGetChildForInvalidDimensionValue() throws IOException { long invalidDimensionValue = Long.MAX_VALUE; - assertNull(starTreeNode.getChildForDimensionValue(invalidDimensionValue, new NumericDimension("field"))); + assertNull(starTreeNode.getChildForDimensionValue(invalidDimensionValue)); } public void testOnlyRootNodePresent() throws IOException { @@ -224,7 +219,7 @@ public void testOnlyRootNodePresent() throws IOException { FixedLengthStarTreeNode starTreeNode = (FixedLengthStarTreeNode) StarTreeFactory.createStarTree(dataIn, starTreeMetadata); assertEquals(starTreeNode.getNumChildren(), 0); - assertNull(starTreeNode.getChildForDimensionValue(randomLong(), new NumericDimension("field"))); + assertNull(starTreeNode.getChildForDimensionValue(randomLong())); assertThrows(IllegalArgumentException.class, () -> starTreeNode.getChildrenIterator().next()); assertThrows(UnsupportedOperationException.class, () -> starTreeNode.getChildrenIterator().remove()); @@ -232,70 +227,6 @@ public void testOnlyRootNodePresent() throws IOException { directory.close(); } - public void testGetChildForUnsignedLongDimensionValue() throws IOException { - - Directory directory = newFSDirectory(createTempDir()); - IndexOutput dataOut = directory.createOutput("star-tree-data-2", IOContext.DEFAULT); - StarTreeWriter starTreeWriter = new StarTreeWriter(); - - InMemoryTreeNode node = new InMemoryTreeNode(0, randomInt(), randomInt(), randomFrom((byte) 0, (byte) -1, (byte) 1), -1); - node.setChildDimensionId(1); - node.setAggregatedDocId(randomInt()); - - InMemoryTreeNode starChild = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) -1, -1); - starChild.setChildDimensionId(-1); - starChild.setAggregatedDocId(randomInt()); - node.addChildNode(starChild, (long) ALL); - - InMemoryTreeNode childWithMinus1 = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) 0, -1); - childWithMinus1.setChildDimensionId(-1); - childWithMinus1.setAggregatedDocId(randomInt()); - - for (int i = 1; i < randomIntBetween(2, 5); i++) { - InMemoryTreeNode child = new InMemoryTreeNode( - node.getDimensionId() + 1, - randomInt(), - randomInt(), - (byte) 0, - node.getDimensionValue() + i - ); - child.setChildDimensionId(-1); - child.setAggregatedDocId(randomInt()); - node.addChildNode(child, child.getDimensionValue()); - } - - node.addChildNode(childWithMinus1, -1L); - - InMemoryTreeNode nullChild = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) 1, -1); - nullChild.setChildDimensionId(-1); - nullChild.setAggregatedDocId(randomInt()); - node.addChildNode(nullChild, null); - - long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, node, 2 + node.getChildren().size(), "star-tree"); - - // asserting on the actual length of the star tree data file - assertEquals(starTreeDataLength, 33L * node.getChildren().size() + 2 * 33); - dataOut.close(); - - IndexInput dataIn = directory.openInput("star-tree-data-2", IOContext.READONCE); - StarTreeMetadata starTreeMetadata = mock(StarTreeMetadata.class); - when(starTreeMetadata.getDataLength()).thenReturn(starTreeDataLength); - when(starTreeMetadata.getDataStartFilePointer()).thenReturn(0L); - - FixedLengthStarTreeNode starTreeNode = (FixedLengthStarTreeNode) StarTreeFactory.createStarTree(dataIn, starTreeMetadata); - - long dimensionValue = -1; - FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue( - dimensionValue, - new UnsignedLongDimension("field") - ); - assertNotNull(childNode); - assertEquals(dimensionValue, childNode.getDimensionValue()); - - dataIn.close(); - directory.close(); - } - public void tearDown() throws Exception { super.tearDown(); dataIn.close(); From 3cd08588aa06385ae001f8117788ff427cc8909c Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Tue, 28 Jan 2025 15:24:05 +0530 Subject: [PATCH 10/10] refractor code --- .../fileformats/node/FixedLengthStarTreeNode.java | 2 -- .../datacube/startree/node/StarTreeNode.java | 1 - .../datacube/startree/StarTreeTestUtils.java | 4 +--- .../builder/StarTreeBuilderFlushFlowTests.java | 14 ++++++++------ .../builder/StarTreeBuilderMergeFlowTests.java | 1 + .../fileformats/data/StarTreeFileFormatsTests.java | 5 +---- 6 files changed, 11 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index 1a75ebf685b24..df2ce9096bfc1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -8,13 +8,11 @@ package org.opensearch.index.compositeindex.datacube.startree.fileformats.node; import org.apache.lucene.store.RandomAccessInput; -import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.Comparator; import java.util.Iterator; /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index c3ae472ab3a58..3767f6850002a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -9,7 +9,6 @@ package org.opensearch.index.compositeindex.datacube.startree.node; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.compositeindex.datacube.Dimension; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 5ede294c98754..7e8ecf79d7443 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -234,9 +234,7 @@ default nodes with this dimension value. We will perform the assertNull() check if (child.getDimensionValue() == -1L) { nodeWithMinusOneValueFound = true; } - assertNotNull( - starTreeNode.getChildForDimensionValue(child.getDimensionValue()) - ); + assertNotNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); } assertStarTreeNode(child, resultChildNode); assertNotEquals(child.getStarTreeNodeType(), StarTreeNodeType.STAR.getValue()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index 39c62590c9a0f..6bc75d01bc9d3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -369,12 +369,14 @@ public void testFlushFlowWithUnsignedLongDimensions() throws IOException { 264 ); -// validateStarTreeFileFormats( -// builder.getRootNode(), -// builder.getStarTreeDocuments().size(), -// starTreeMetadata, -// builder.getStarTreeDocuments() -// ); + // validateStarTreeFileFormats( + // builder.getRootNode(), + // builder.getStarTreeDocuments().size(), + // starTreeMetadata, + // builder.getStarTreeDocuments() + // ); + + // TODO: Fix this post 2.19 [Handling search for unsigned-long as part of star-tree] } public void testFlushFlowBuild() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index 0d61ba9267625..22460d6ba9f68 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -429,6 +429,7 @@ public void testMergeFlowForUnsignedLong() throws IOException { // starTreeMetadata, // builder.getStarTreeDocuments() // ); + // TODO: Fix this post 2.19 [Handling search for unsigned-long as part of star-tree] } public void testMergeFlow_randomNumberTypes() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java index 68ecf3ceee642..4d2aa5eaf78cf 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; @@ -122,9 +121,7 @@ public void test_starTreeSearch() throws IOException { for (int i = 0; i < maxLevels - 1; i++) { InMemoryTreeNode randomChildNode = randomFrom(inMemoryTreeNode.getChildren().values()); - StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue( - randomChildNode.getDimensionValue() - ); + StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue(randomChildNode.getDimensionValue()); assertNotNull(randomStarTreeChildNode); assertStarTreeNode(randomStarTreeChildNode, randomChildNode);