Skip to content

Enhance the cost computing mechanism and push down context #4353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LantaoJin merged 16 commits into from
Sep 28, 2025
Merged

Enhance the cost computing mechanism and push down context #4353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
da16493
Make PushDownContext support caching the built request builder
qianheng-aws Sep 22, 2025
061d74b
Move PushDownContext outside of the AbstractCalciteIndexScan
qianheng-aws Sep 22, 2025
c35062a
Refactor cost computing
qianheng-aws Sep 23, 2025
20981cf
migrate explain IT from json format to yaml format
qianheng-aws Sep 23, 2025
493164e
Fix opensearch request reuse by equivalent operator
qianheng-aws Sep 24, 2025
140af5c
Merge remote-tracking branch 'refs/remotes/origin/main' into costRefact
qianheng-aws Sep 24, 2025
5c83bb0
fix IT after merging main
qianheng-aws Sep 24, 2025
6544a62
remove plans assert in execution IT
qianheng-aws Sep 24, 2025
a8d78ee
Merge remote-tracking branch 'refs/remotes/origin/main' into costRefact
qianheng-aws Sep 24, 2025
7ded561
Merging main
qianheng-aws Sep 24, 2025
63d9774
Add UT for cost evaluation of AbstractCalciteIndexScan
qianheng-aws Sep 24, 2025
cc64b05
Merge remote-tracking branch 'refs/remotes/origin/main' into costRefact
qianheng-aws Sep 26, 2025
820aee0
merging main
qianheng-aws Sep 26, 2025
9728a5a
merging main
qianheng-aws Sep 26, 2025
a10714c
Merge remote-tracking branch 'refs/remotes/origin/main' into costRefact
qianheng-aws Sep 26, 2025
7f6c308
merging main
qianheng-aws Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;

Expand All @@ -27,7 +26,6 @@ public class YamlFormatter {
YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings
yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR);
YAML_MAPPER = new ObjectMapper(yamlFactory);
YAML_MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
Copy link
Collaborator Author

@qianheng-aws qianheng-aws Sep 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove this config, otherwise for the plan of v2 which contains multi-children for one level, it will put the children at beginning and the operator name at the end

}

/**
Expand Down
40 changes: 20 additions & 20 deletions integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,19 +270,19 @@ public void testExplainWithTimechartAvg() throws IOException {
var result = explainQueryToString("source=events | timechart span=1m avg(cpu_usage) by host");
String expected =
!isPushdownDisabled()
? loadFromFile("expectedOutput/calcite/explain_timechart.json")
: loadFromFile("expectedOutput/calcite/explain_timechart_no_pushdown.json");
assertJsonEqualsIgnoreId(expected, result);
? loadFromFile("expectedOutput/calcite/explain_timechart.yaml")
: loadFromFile("expectedOutput/calcite/explain_timechart_no_pushdown.yaml");
assertYamlEqualsJsonIgnoreId(expected, result);
}

@Test
public void testExplainWithTimechartCount() throws IOException {
var result = explainQueryToString("source=events | timechart span=1m count() by host");
String expected =
!isPushdownDisabled()
? loadFromFile("expectedOutput/calcite/explain_timechart_count.json")
: loadFromFile("expectedOutput/calcite/explain_timechart_count_no_pushdown.json");
assertJsonEqualsIgnoreId(expected, result);
? loadFromFile("expectedOutput/calcite/explain_timechart_count.yaml")
: loadFromFile("expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml");
assertYamlEqualsJsonIgnoreId(expected, result);
}

@Test
Expand Down Expand Up @@ -335,8 +335,8 @@ public void testExplainStatsWithBinsOnTimeField() throws IOException {

@Test
public void testExplainBinWithSpan() throws IOException {
String expected = loadExpectedPlan("explain_bin_span.json");
assertJsonEqualsIgnoreId(
String expected = loadExpectedPlan("explain_bin_span.yaml");
assertYamlEqualsJsonIgnoreId(
expected,
explainQueryToString(
"source=opensearch-sql_test_index_account | bin age span=10 | head 5"));
Expand All @@ -362,8 +362,8 @@ public void testExplainBinWithStartEnd() throws IOException {

@Test
public void testExplainBinWithAligntime() throws IOException {
String expected = loadExpectedPlan("explain_bin_aligntime.json");
assertJsonEqualsIgnoreId(
String expected = loadExpectedPlan("explain_bin_aligntime.yaml");
assertYamlEqualsJsonIgnoreId(
expected,
explainQueryToString(
"source=opensearch-sql_test_index_time_data | bin @timestamp span=2h aligntime=latest |"
Expand Down Expand Up @@ -413,8 +413,8 @@ public void testEventstatsDistinctCountFunctionExplain() throws IOException {
// Only for Calcite, as v2 gets unstable serialized string for function
@Test
public void testExplainOnAggregationWithSumEnhancement() throws IOException {
String expected = loadExpectedPlan("explain_agg_with_sum_enhancement.json");
assertJsonEqualsIgnoreId(
String expected = loadExpectedPlan("explain_agg_with_sum_enhancement.yaml");
assertYamlEqualsJsonIgnoreId(
expected,
explainQueryToString(
String.format(
Expand Down Expand Up @@ -545,16 +545,16 @@ public void testRegexExplain() throws IOException {
String query =
"source=opensearch-sql_test_index_account | regex lastname='^[A-Z][a-z]+$' | head 5";
var result = explainQueryToString(query);
String expected = loadExpectedPlan("explain_regex.json");
assertJsonEqualsIgnoreId(expected, result);
String expected = loadExpectedPlan("explain_regex.yaml");
assertYamlEqualsJsonIgnoreId(expected, result);
}

@Test
public void testRegexNegatedExplain() throws IOException {
String query = "source=opensearch-sql_test_index_account | regex lastname!='.*son$' | head 5";
var result = explainQueryToString(query);
String expected = loadExpectedPlan("explain_regex_negated.json");
assertJsonEqualsIgnoreId(expected, result);
String expected = loadExpectedPlan("explain_regex_negated.yaml");
assertYamlEqualsJsonIgnoreId(expected, result);
}

@Test
Expand All @@ -581,8 +581,8 @@ public void testRexExplain() throws IOException {
"source=opensearch-sql_test_index_account | rex field=lastname \\\"(?<initial>^[A-Z])\\\" |"
+ " head 5";
var result = explainQueryToString(query);
String expected = loadExpectedPlan("explain_rex.json");
assertJsonEqualsIgnoreId(expected, result);
String expected = loadExpectedPlan("explain_rex.yaml");
assertYamlEqualsJsonIgnoreId(expected, result);
}

@Test
Expand Down Expand Up @@ -628,8 +628,8 @@ public void testPushdownLimitIntoAggregation() throws IOException {
expected,
explainQueryToString("source=opensearch-sql_test_index_account | stats count() by state"));

expected = loadExpectedPlan("explain_limit_agg_pushdown2.json");
assertJsonEqualsIgnoreId(
expected = loadExpectedPlan("explain_limit_agg_pushdown2.yaml");
assertYamlEqualsJsonIgnoreId(
expected,
explainQueryToString(
"source=opensearch-sql_test_index_account | stats count() by state | head 100"));
Expand Down
1 change: 1 addition & 0 deletions integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class CalcitePPLExplainIT extends PPLIntegTestCase {

@Override
public void init() throws Exception {
GlobalPushdownConfig.enabled = false;
super.init();
enableCalcite();

Expand Down
65 changes: 0 additions & 65 deletions integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,6 @@ public void testPushdownSortPlusExpression() throws IOException {
Locale.ROOT,
"source=%s | eval age2 = age + 2 | sort age2 | fields age | head 2",
TEST_INDEX_BANK);
String explained = explainQueryToString(ppl);
if (!isPushdownDisabled()) {
assertTrue(
explained.contains(
"[SORT->[{\\n"
+ " \\\"age\\\" : {\\n"
+ " \\\"order\\\" : \\\"asc\\\",\\n"
+ " \\\"missing\\\" : \\\"_first\\\"\\n"
+ " }\\n"
+ "}]"));
}

JSONObject result = executeQuery(ppl);
verifyOrder(result, rows(28), rows(32));
}
Expand All @@ -62,18 +50,6 @@ public void testPushdownSortMinusExpression() throws IOException {
Locale.ROOT,
"source=%s | eval age2 = 1 - age | sort age2 | fields age | head 2",
TEST_INDEX_BANK);
String explained = explainQueryToString(ppl);
if (!isPushdownDisabled()) {
assertTrue(
explained.contains(
"[SORT->[{\\n"
+ " \\\"age\\\" : {\\n"
+ " \\\"order\\\" : \\\"desc\\\",\\n"
+ " \\\"missing\\\" : \\\"_first\\\"\\n"
+ " }\\n"
+ "}]"));
}

JSONObject result = executeQuery(ppl);
verifyOrder(result, rows(39), rows(36));
}
Expand All @@ -85,18 +61,6 @@ public void testPushdownSortTimesExpression() throws IOException {
Locale.ROOT,
"source=%s | eval age2 = 5 * age | sort age2 | fields age | head 2",
TEST_INDEX_BANK);
String explained = explainQueryToString(ppl);
if (!isPushdownDisabled()) {
assertTrue(
explained.contains(
"[SORT->[{\\n"
+ " \\\"age\\\" : {\\n"
+ " \\\"order\\\" : \\\"asc\\\",\\n"
+ " \\\"missing\\\" : \\\"_first\\\"\\n"
+ " }\\n"
+ "}]"));
}

JSONObject result = executeQuery(ppl);
verifyOrder(result, rows(28), rows(32));
}
Expand All @@ -108,23 +72,6 @@ public void testPushdownSortByMultiExpressions() throws IOException {
Locale.ROOT,
"source=%s | eval age2 = 5 * age | sort gender, age2 | fields gender, age | head 2",
TEST_INDEX_BANK);
String explained = explainQueryToString(ppl);
if (!isPushdownDisabled()) {
assertTrue(
explained.contains(
"[SORT->[{\\n"
+ " \\\"gender.keyword\\\" : {\\n"
+ " \\\"order\\\" : \\\"asc\\\",\\n"
+ " \\\"missing\\\" : \\\"_first\\\"\\n"
+ " }\\n"
+ "}, {\\n"
+ " \\\"age\\\" : {\\n"
+ " \\\"order\\\" : \\\"asc\\\",\\n"
+ " \\\"missing\\\" : \\\"_first\\\"\\n"
+ " }\\n"
+ "}]"));
}

JSONObject result = executeQuery(ppl);
verifyOrder(result, rows("F", 28), rows("F", 34));
}
Expand All @@ -136,18 +83,6 @@ public void testPushdownSortCastExpression() throws IOException {
Locale.ROOT,
"source=%s | eval age2 = cast(age * 5 as long) | sort age2 | fields age | head 2",
TEST_INDEX_BANK);
String explained = explainQueryToString(ppl);
if (!isPushdownDisabled()) {
assertTrue(
explained.contains(
"[SORT->[{\\n"
+ " \\\"age\\\" : {\\n"
+ " \\\"order\\\" : \\\"asc\\\",\\n"
+ " \\\"missing\\\" : \\\"_first\\\"\\n"
+ " }\\n"
+ "}]"));
}

JSONObject result = executeQuery(ppl);
verifyOrder(result, rows(28), rows(32));
}
Expand Down
Loading
Loading