Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import java.util.Map;
import org.openjdk.jmh.annotations.Benchmark;
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
import org.opensearch.sql.opensearch.util.MergeRules.MergeRuleHelper;

public class MergeArrayAndObjectMapBenchmark {
private static final List<Map<String, OpenSearchDataType>> candidateMaps = prepareListOfMaps(120);
Expand All @@ -21,7 +21,7 @@ public class MergeArrayAndObjectMapBenchmark {
public void testMerge() {
Map<String, OpenSearchDataType> finalResult = new HashMap<>();
for (Map<String, OpenSearchDataType> map : candidateMaps) {
OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map);
MergeRuleHelper.merge(finalResult, map);

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not related. just fix a compile error in benchmarks module.

}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,23 @@ public void supportSearchSargPushDown_multiRange() throws IOException {
// Only for Calcite
@Test
public void supportSearchSargPushDown_timeRange() throws IOException {
String query =
"source=opensearch-sql_test_index_bank"
+ "| where birthdate >= '2016-12-08 00:00:00.000000000' "
+ "and birthdate < '2018-11-09 00:00:00.000000000'";
var result = explainQueryToString(query);
String expected = loadExpectedPlan("explain_sarg_filter_push_time_range.json");
assertJsonEqualsIgnoreId(
expected,
explainQueryToString(
"source=opensearch-sql_test_index_bank"
+ "| where birthdate >= '2016-12-08 00:00:00.000000000' "
+ "and birthdate < '2018-11-09 00:00:00.000000000' "));
assertJsonEqualsIgnoreId(expected, result);
}

// Only for Calcite
@Test
public void supportPushDownSortMergeJoin() throws IOException {
String query =
"source=opensearch-sql_test_index_bank| join left=l right=r on"
+ " l.account_number=r.account_number opensearch-sql_test_index_bank";
var result = explainQueryToString(query);
String expected = loadExpectedPlan("explain_merge_join_sort_push.json");
assertJsonEqualsIgnoreId(expected, result);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ public void aggregationFunctionInHaving() throws IOException {

JSONArray dataRows = getDataRows(response);
assertEquals(1, dataRows.length());
assertEquals("m", dataRows.getJSONArray(0).getString(0));
assertEquals("M", dataRows.getJSONArray(0).getString(0));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public void testRareWithGroup() throws IOException {
rows("F", "OK", 7),
rows("F", "KS", 7),
rows("F", "CO", 7),
rows("F", "NV", 8),
rows("F", "AR", 8),
rows("M", "NE", 5),
rows("M", "RI", 5),
rows("M", "NV", 5),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.schema;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder;
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder;

Expand Down Expand Up @@ -358,27 +357,15 @@ public void testStatsBySpanAndMultipleFields() throws IOException {
schema("span(age,10)", null, "int"),
schema("gender", null, "string"),
schema("state", null, "string"));
if (isCalciteEnabled()) {
verifyDataRows(
response,
rows(1, 20, "F", "VA"),
rows(1, 30, "F", "IN"),
rows(1, 30, "F", "PA"),
rows(1, 30, "M", "IL"),
rows(1, 30, "M", "MD"),
rows(1, 30, "M", "TN"),
rows(1, 30, "M", "WA"));
} else {
verifyDataRowsInOrder(
response,
rows(1, 20, "f", "VA"),
rows(1, 30, "f", "IN"),
rows(1, 30, "f", "PA"),
rows(1, 30, "m", "IL"),
rows(1, 30, "m", "MD"),
rows(1, 30, "m", "TN"),
rows(1, 30, "m", "WA"));
Comment on lines -374 to -380

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this lower case values in v2 were caused by grouping by a fielddata field.

}
verifyDataRows(
response,
rows(1, 20, "F", "VA"),
rows(1, 30, "F", "IN"),
rows(1, 30, "F", "PA"),
rows(1, 30, "M", "IL"),
rows(1, 30, "M", "MD"),
rows(1, 30, "M", "TN"),
rows(1, 30, "M", "WA"));
}

@Test
Expand All @@ -395,27 +382,15 @@ public void testStatsByMultipleFieldsAndSpan() throws IOException {
schema("span(age,10)", null, "int"),
schema("gender", null, "string"),
schema("state", null, "string"));
if (isCalciteEnabled()) {
verifyDataRows(
response,
rows(1, 20, "F", "VA"),
rows(1, 30, "F", "IN"),
rows(1, 30, "F", "PA"),
rows(1, 30, "M", "IL"),
rows(1, 30, "M", "MD"),
rows(1, 30, "M", "TN"),
rows(1, 30, "M", "WA"));
} else {
verifyDataRowsInOrder(
response,
rows(1, 20, "f", "VA"),
rows(1, 30, "f", "IN"),
rows(1, 30, "f", "PA"),
rows(1, 30, "m", "IL"),
rows(1, 30, "m", "MD"),
rows(1, 30, "m", "TN"),
rows(1, 30, "m", "WA"));
}
verifyDataRows(
response,
rows(1, 20, "F", "VA"),
rows(1, 30, "F", "IN"),
rows(1, 30, "F", "PA"),
rows(1, 30, "M", "IL"),
rows(1, 30, "M", "MD"),
rows(1, 30, "M", "TN"),
rows(1, 30, "M", "WA"));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"calcite": {
"logical": "LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[inner])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n",
"physical": "EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"calcite": {
"logical": "LogicalSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender], SORT->[{\n \"balance\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"gender\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"address\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender], SORT->[{\n \"balance\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"gender.subfield1\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"address.subfield2\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.subfield1\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"address.subfield2\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"calcite": {
"logical": "LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[inner])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n",
"physical": "EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,12 @@
"description": {
"fields": "[account_number, firstname, address, balance, gender]"
},
"children": [
{
"name": "OpenSearchIndexScan",
"description": {
"request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=s9y3QQEhb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9hY2NvdW50Fndla1VpMi1kVHh5Qi1lYnhPQnlSbXcAFkU4Qm9UVURIUWI2a3pjNkhmQkxvc2cAAAAAAAAAAAEWdTZkUG5oYkZTWWl2UnpFVXlhNmhXZwEWd2VrVWkyLWRUeHlCLWVieE9CeVJtdwAA, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
},
"children": []
}
]
"children": [{
"name": "OpenSearchIndexScan",
"description": {
"request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"address.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.keyword\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"address.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
},
"children": []
}]
}
}
}
Loading
Loading