Skip to content

Commit

Permalink
Changes to handle count and avg metrics as part of star tree mapping
Browse files Browse the repository at this point in the history 
Signed-off-by: Bharathwaj G <[email protected]>
  • Loading branch information
@bharath-techie
bharath-techie committed Aug 7, 2024
1 parent a918530 commit e09106d
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 49 deletions.
20 changes: 6 additions & 14 deletions server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,13 +265,9 @@ public void testValidCompositeIndex() {
assertEquals(expectedTimeUnits, dateDim.getIntervals());
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(
MetricStat.AVG,
MetricStat.COUNT,
MetricStat.SUM,
MetricStat.MAX,
MetricStat.MIN
);

// Assert default metrics
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(
Expand Down Expand Up @@ -349,13 +345,9 @@ public void testUpdateIndexWhenMappingIsSame() {
assertEquals(expectedTimeUnits, dateDim.getIntervals());
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(
MetricStat.AVG,
MetricStat.COUNT,
MetricStat.SUM,
MetricStat.MAX,
MetricStat.MIN
);

// Assert default metrics
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(
Expand Down
42 changes: 36 additions & 6 deletions server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,59 @@

import org.opensearch.common.annotation.ExperimentalApi;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;

/**
* Supported metric types for composite index
*
* @opensearch.experimental
*/
@ExperimentalApi
public enum MetricStat {
COUNT("count"),
AVG("avg"),
SUM("sum"),
MIN("min"),
MAX("max");
COUNT("count", null),
SUM("sum", null),
MIN("min", null),
MAX("max", null),
AVG("avg", new MetricStat[] { COUNT, SUM });

private final String typeName;
private final MetricStat[] derivedFrom;

MetricStat(String typeName) {
MetricStat(String typeName, MetricStat[] derivedFrom) {
this.typeName = typeName;
this.derivedFrom = derivedFrom;
}

public String getTypeName() {
return typeName;
}

/**
* Return the list of metrics that this metric is derived from
* For example, AVG is derived from COUNT and SUM
*/
public List<MetricStat> getDerivedFromMetrics() {
return Arrays.asList(derivedFrom);
}

/**
* Return true if this metric is derived from other metrics
* For example, AVG is derived from COUNT and SUM
*/
public boolean isDerivedMetric() {
return derivedFrom != null;
}

/**
* Return required metrics for every metric field in star tree field
*/
public static Set<MetricStat> getRequiredMetrics() {
return Collections.singleton(MetricStat.COUNT);
}

public static MetricStat fromTypeName(String typeName) {
for (MetricStat metric : MetricStat.values()) {
if (metric.getTypeName().equalsIgnoreCase(typeName)) {
Expand Down
13 changes: 4 additions & 9 deletions ...ain/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import java.util.Arrays;
import java.util.List;
import java.util.function.Function;

/**
* Index settings for star tree fields. The settings are final as right now
Expand Down Expand Up @@ -93,16 +94,10 @@ public class StarTreeIndexSettings {
/**
* Default metrics for metrics as part of star tree fields
*/
public static final Setting<List<MetricStat>> DEFAULT_METRICS_LIST = Setting.listSetting(
public static final Setting<List<String>> DEFAULT_METRICS_LIST = Setting.listSetting(
"index.composite_index.star_tree.field.default.metrics",
Arrays.asList(
MetricStat.AVG.toString(),
MetricStat.COUNT.toString(),
MetricStat.SUM.toString(),
MetricStat.MAX.toString(),
MetricStat.MIN.toString()
),
MetricStat::fromTypeName,
Arrays.asList(MetricStat.COUNT.toString(), MetricStat.SUM.toString()),
Function.identity(),
Setting.Property.IndexScope,
Setting.Property.Final
);
Expand Down
17 changes: 11 additions & 6 deletions server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,14 +262,19 @@ private Metric getMetric(String name, Map<String, Object> metric, Mapper.TypePar
.collect(Collectors.toList());
metric.remove(STATS);
if (metricStrings.isEmpty()) {
metricTypes = new ArrayList<>(StarTreeIndexSettings.DEFAULT_METRICS_LIST.get(context.getSettings()));
} else {
Set<MetricStat> metricSet = new LinkedHashSet<>();
for (String metricString : metricStrings) {
metricSet.add(MetricStat.fromTypeName(metricString));
metricStrings = new ArrayList<>(StarTreeIndexSettings.DEFAULT_METRICS_LIST.get(context.getSettings()));
}
// Add all required field initially
Set<MetricStat> metricSet = new LinkedHashSet<>(MetricStat.getRequiredMetrics());
for (String metricString : metricStrings) {
MetricStat metricStat = MetricStat.fromTypeName(metricString);
if (metricStat.isDerivedMetric()) {
metricSet.addAll(metricStat.getDerivedFromMetrics());
} else {
metricSet.add(metricStat);
}
metricTypes = new ArrayList<>(metricSet);
}
metricTypes = new ArrayList<>(metricSet);
return new Metric(name, metricTypes);
}

Expand Down
105 changes: 91 additions & 14 deletions server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public void teardown() {
}

public void testValidStarTree() throws IOException {
MapperService mapperService = createMapperService(getExpandedMapping("status", "size"));
MapperService mapperService = createMapperService(getExpandedMappingWithJustAvg("status", "size"));
Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
for (CompositeMappedFieldType type : compositeFieldTypes) {
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type;
Expand All @@ -66,7 +66,37 @@ public void testValidStarTree() throws IOException {
assertEquals(expectedTimeUnits, dateDim.getIntervals());
assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("size", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.SUM, MetricStat.AVG);

// Assert COUNT and SUM gets added when AVG is mentioned
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
assertEquals(
new HashSet<>(Arrays.asList("@timestamp", "status")),
starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()
);
}
}

public void testMetricsWithJustSum() throws IOException {
MapperService mapperService = createMapperService(getExpandedMappingWithJustSum("status", "size"));
Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes();
for (CompositeMappedFieldType type : compositeFieldTypes) {
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type;
assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField());
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
Rounding.DateTimeUnit.DAY_OF_MONTH,
Rounding.DateTimeUnit.MONTH_OF_YEAR
);
assertEquals(expectedTimeUnits, dateDim.getIntervals());
assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("size", starTreeFieldType.getMetrics().get(0).getField());

// Assert COUNT gets added for any metric field
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
Expand All @@ -92,13 +122,7 @@ public void testValidStarTreeDefaults() throws IOException {
assertEquals(expectedTimeUnits, dateDim.getIntervals());
assertEquals("status", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("status", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(
MetricStat.AVG,
MetricStat.COUNT,
MetricStat.SUM,
MetricStat.MAX,
MetricStat.MIN
);
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.COUNT, MetricStat.SUM);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode());
Expand All @@ -109,15 +133,15 @@ public void testValidStarTreeDefaults() throws IOException {
public void testInvalidDim() {
MapperParsingException ex = expectThrows(
MapperParsingException.class,
() -> createMapperService(getExpandedMapping("invalid", "size"))
() -> createMapperService(getExpandedMappingWithJustAvg("invalid", "size"))
);
assertEquals("Failed to parse mapping [_doc]: unknown dimension field [invalid]", ex.getMessage());
}

public void testInvalidMetric() {
MapperParsingException ex = expectThrows(
MapperParsingException.class,
() -> createMapperService(getExpandedMapping("status", "invalid"))
() -> createMapperService(getExpandedMappingWithJustAvg("status", "invalid"))
);
assertEquals("Failed to parse mapping [_doc]: unknown metric field [invalid]", ex.getMessage());
}
Expand Down Expand Up @@ -232,6 +256,10 @@ public void testMetric() {
assertEquals(MetricStat.MIN, MetricStat.fromTypeName("min"));
assertEquals(MetricStat.SUM, MetricStat.fromTypeName("sum"));
assertEquals(MetricStat.AVG, MetricStat.fromTypeName("avg"));

assertEquals(Set.of(MetricStat.COUNT), MetricStat.getRequiredMetrics());
assertEquals(List.of(MetricStat.COUNT, MetricStat.SUM), MetricStat.AVG.getDerivedFromMetrics());

IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> MetricStat.fromTypeName("invalid"));
assertEquals("Invalid metric stat: invalid", ex.getMessage());
}
Expand Down Expand Up @@ -310,7 +338,7 @@ public void testStarTreeField() {
}

public void testValidations() throws IOException {
MapperService mapperService = createMapperService(getExpandedMapping("status", "size"));
MapperService mapperService = createMapperService(getExpandedMappingWithJustAvg("status", "size"));
Settings settings = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build();
CompositeIndexSettings enabledCompositeIndexSettings = new CompositeIndexSettings(
settings,
Expand Down Expand Up @@ -370,7 +398,7 @@ public void testValidations() throws IOException {
);
}

private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException {
private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) throws IOException {
return topMapping(b -> {
b.startObject("composite");
b.startObject("startree");
Expand Down Expand Up @@ -399,7 +427,6 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE
b.startObject();
b.field("name", metric);
b.startArray("stats");
b.value("sum");
b.value("avg");
b.endArray();
b.endObject();
Expand All @@ -421,6 +448,56 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE
});
}

private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) throws IOException {
return topMapping(b -> {
b.startObject("composite");
b.startObject("startree");
b.field("type", "star_tree");
b.startObject("config");
b.field("max_leaf_docs", 100);
b.startArray("skip_star_node_creation_for_dimensions");
{
b.value("@timestamp");
b.value("status");
}
b.endArray();
b.startArray("ordered_dimensions");
b.startObject();
b.field("name", "@timestamp");
b.startArray("calendar_intervals");
b.value("day");
b.value("month");
b.endArray();
b.endObject();
b.startObject();
b.field("name", dim);
b.endObject();
b.endArray();
b.startArray("metrics");
b.startObject();
b.field("name", metric);
b.startArray("stats");
b.value("sum");
b.endArray();
b.endObject();
b.endArray();
b.endObject();
b.endObject();
b.endObject();
b.startObject("properties");
b.startObject("@timestamp");
b.field("type", "date");
b.endObject();
b.startObject("status");
b.field("type", "integer");
b.endObject();
b.startObject("size");
b.field("type", "integer");
b.endObject();
b.endObject();
});
}

private XContentBuilder getMinMapping() throws IOException {
return getMinMapping(false, false, false, false);
}
Expand Down

0 comments on commit e09106d

Please sign in to comment.