From de45f085beacebded4d77ae63432a9335427904c Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Fri, 21 Feb 2025 12:40:05 +0800 Subject: [PATCH 1/5] Build integration test framework Signed-off-by: Lantao Jin --- async-query-core/build.gradle | 16 +- .../execution/statestore/StateStore.java | 2 +- .../flint/OpenSearchFlintIndexClient.java | 2 +- build.gradle | 14 +- .../sql/common/setting/Settings.java | 1 + .../sql/calcite/CalciteRelNodeVisitor.java | 19 +- .../sql/calcite/plan/OpenSearchTable.java | 4 +- ...aTypes.java => OpenSearchTypeFactory.java} | 36 +- .../sql/data/model/ExprValueUtils.java | 13 +- .../opensearch/sql/executor/QueryService.java | 7 + .../sql/expression/datetime/ExtractTest.java | 2 +- doctest/build.gradle | 4 +- integ-test/build.gradle | 16 +- .../calcite/remote/CalciteCsvFormatIT.java | 18 + .../sql/calcite/remote/CalciteDataTypeIT.java | 35 ++ .../remote/CalciteFieldsCommandIT.java | 42 ++ .../calcite/remote/CalciteHeadCommandIT.java | 18 + .../calcite/remote/CalciteLikeQueryIT.java | 20 + .../calcite/remote/CalcitePPLPluginIT.java | 12 + .../remote/CalciteQueryAnalysisIT.java | 39 ++ .../remote/CalciteSearchCommandIT.java | 28 + .../calcite/remote/CalciteSortCommandIT.java | 25 + .../calcite/remote/CalciteStatsCommandIT.java | 22 + .../remote/CalciteUnsupportedCommandIT.java | 30 + .../calcite/remote/CalciteWhereCommandIT.java | 21 + .../CalcitePPLAggregationIT.java | 7 +- .../{ => standalone}/CalcitePPLBasicIT.java | 300 ++++------ .../calcite/standalone/CalcitePPLSortIT.java | 552 ++++++++++++++++++ .../CalcitePPLTestCase.java} | 17 +- .../opensearch/sql/ppl/PPLIntegTestCase.java | 33 ++ .../sql/legacy/domain/SearchResult.java | 4 +- .../format/PrettyFormatRestExecutor.java | 2 +- .../executor/format/SelectResultSet.java | 3 +- .../legacy/executor/join/ElasticUtils.java | 10 +- .../join/NestedLoopsElasticExecutor.java | 2 +- .../executor/OpenSearchExecutionEngine.java | 10 +- .../physical/EnumerableIndexScanRule.java | 55 ++ .../setting/OpenSearchSettings.java | 14 + .../opensearch/storage/OpenSearchIndex.java | 4 +- .../storage/scan/CalciteLogicalTableScan.java | 45 ++ .../scan/CalciteOpenSearchIndexScan.java | 2 +- .../scan/OpenSearchIndexEnumerator.java | 4 +- ...a => JdbcOpenSearchDataTypeConvertor.java} | 3 +- plugin/build.gradle | 2 +- .../sql/ppl/calcite/CalcitePPLBasicTest.java | 1 + 45 files changed, 1256 insertions(+), 260 deletions(-) rename core/src/main/java/org/opensearch/sql/calcite/utils/{OpenSearchRelDataTypes.java => OpenSearchTypeFactory.java} (79%) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteCsvFormatIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteHeadCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPluginIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnsupportedCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java rename integ-test/src/test/java/org/opensearch/sql/calcite/{ => standalone}/CalcitePPLAggregationIT.java (98%) rename integ-test/src/test/java/org/opensearch/sql/calcite/{ => standalone}/CalcitePPLBasicIT.java (78%) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java rename integ-test/src/test/java/org/opensearch/sql/calcite/{CalcitePPLIntegTestCase.java => standalone/CalcitePPLTestCase.java} (93%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalTableScan.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/util/{JdbcUtil.java => JdbcOpenSearchDataTypeConvertor.java} (95%) diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index 37bf6748c9d..2c0f8292aab 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -141,11 +141,11 @@ jacocoTestCoverageVerification { } check.dependsOn jacocoTestCoverageVerification -shadowJar { - archiveBaseName.set('async-query-core') - archiveVersion.set('1.0.0') // Set the desired version - archiveClassifier.set('all') - - from sourceSets.main.output - configurations = [project.configurations.runtimeClasspath] -} \ No newline at end of file +//shadowJar { +// archiveBaseName.set('async-query-core') +// archiveVersion.set('1.0.0') // Set the desired version +// archiveClassifier.set('all') +// +// from sourceSets.main.output +// configurations = [project.configurations.runtimeClasspath] +//} \ No newline at end of file diff --git a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java index 552c646cbe0..41f41802e0a 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java @@ -260,7 +260,7 @@ public long count(String indexName, QueryBuilder query) { throw new RuntimeException( "Fetching job metadata information failed with status : " + searchResponse.status()); } else { - return searchResponse.getHits().getTotalHits().value; + return searchResponse.getHits().getTotalHits().value(); } } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java index 7a655f0678a..db29921c089 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; @RequiredArgsConstructor diff --git a/build.gradle b/build.gradle index a7900b3075d..6b7af6ada43 100644 --- a/build.gradle +++ b/build.gradle @@ -119,11 +119,15 @@ allprojects { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.9" - resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:5.3.1" - resolutionStrategy.force 'org.apache.httpcomponents.core5:httpcore5:5.2.5' - resolutionStrategy.force 'org.apache.httpcomponents.core5:httpcore5-h2:5.2.5' - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-annotations:2.17.2' - resolutionStrategy.force 'com.fasterxml.jackson:jackson-bom:2.17.2' + resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" + resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" + resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-annotations:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.18.2" resolutionStrategy.force 'com.google.protobuf:protobuf-java:3.25.5' resolutionStrategy.force 'org.locationtech.jts:jts-core:1.19.0' resolutionStrategy.force 'com.google.errorprone:error_prone_annotations:2.28.0' diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index a39989ed3cf..63ee60b7683 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -30,6 +30,7 @@ public enum Key { /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), + CALCITE_FALLBACK_ALLOWED("plugins.calcite.fallback.allowed"), /** Query Settings. */ FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"), diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 208228628c8..75fd8fa5b32 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -92,13 +92,20 @@ public RelNode visitFilter(Filter node, CalcitePlanContext context) { @Override public RelNode visitProject(Project node, CalcitePlanContext context) { visitChildren(node, context); - List projectList = - node.getProjectList().stream() - .filter(expr -> !(expr instanceof AllFields)) - .map(expr -> rexVisitor.analyze(expr, context)) - .collect(Collectors.toList()); - if (projectList.isEmpty()) { + List projectList; + if (node.getProjectList().stream().anyMatch(e -> e instanceof AllFields)) { + // if (context.relBuilder.peek() instanceof TableScan) { + // // force add project all to stack to avoid the TableScan in plan only. + // // if the stack only contains OpenSearchTableScan.ENUMERABLE, it will fail + // // e.g. search source=table should return project(all_fields, child=tableScan) + // context.relBuilder.project(context.relBuilder.fields(), ImmutableList.of(), true); + // } return context.relBuilder.peek(); + } else { + projectList = + node.getProjectList().stream() + .map(expr -> rexVisitor.analyze(expr, context)) + .collect(Collectors.toList()); } if (node.isExcluded()) { context.relBuilder.projectExcept(projectList); diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchTable.java b/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchTable.java index 92891ede84f..79e9a0d54c4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchTable.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchTable.java @@ -16,7 +16,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Schemas; import org.apache.calcite.schema.TranslatableTable; -import org.opensearch.sql.calcite.utils.OpenSearchRelDataTypes; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; public abstract class OpenSearchTable extends AbstractQueryableTable implements TranslatableTable, org.opensearch.sql.storage.Table { @@ -27,7 +27,7 @@ protected OpenSearchTable(Type elementType) { @Override public RelDataType getRowType(RelDataTypeFactory relDataTypeFactory) { - return OpenSearchRelDataTypes.convertSchema(this); + return OpenSearchTypeFactory.convertSchema(this); } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchRelDataTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java similarity index 79% rename from core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchRelDataTypes.java rename to core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 04c7b29d451..0dfa7d65f78 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchRelDataTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -16,11 +16,12 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.storage.Table; -public class OpenSearchRelDataTypes extends JavaTypeFactoryImpl { - public static final OpenSearchRelDataTypes TYPE_FACTORY = - new OpenSearchRelDataTypes(RelDataTypeSystem.DEFAULT); +/** This class is used to create RelDataType and map RelDataType to Java data type */ +public class OpenSearchTypeFactory extends JavaTypeFactoryImpl { + public static final OpenSearchTypeFactory TYPE_FACTORY = + new OpenSearchTypeFactory(RelDataTypeSystem.DEFAULT); - private OpenSearchRelDataTypes(RelDataTypeSystem typeSystem) { + private OpenSearchTypeFactory(RelDataTypeSystem typeSystem) { super(typeSystem); } @@ -108,8 +109,33 @@ public static RelDataType convertSchema(Table table) { List typeList = new ArrayList<>(); for (Map.Entry entry : table.getFieldTypes().entrySet()) { fieldNameList.add(entry.getKey()); - typeList.add(OpenSearchRelDataTypes.convertSchemaField(entry.getValue())); + typeList.add(OpenSearchTypeFactory.convertSchemaField(entry.getValue())); } return TYPE_FACTORY.createStructType(typeList, fieldNameList, true); } + // + // @Override + // public Type getJavaClass(RelDataType type) { + // if (type instanceof JavaType) { + // JavaType javaType = (JavaType) type; + // return javaType.getJavaClass(); + // } + // if (type instanceof BasicSqlType || type instanceof IntervalSqlType) { + // switch (type.getSqlTypeName()) { + // case DATE: + // return Date.class; + // case TIME: + // case TIME_WITH_LOCAL_TIME_ZONE: + // case TIME_TZ: + // return Time.class; + // case TIMESTAMP: + // case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + // case TIMESTAMP_TZ: + // return Timestamp.class; + // default: + // break; + // } + // } + // return super.getJavaClass(type); + // } } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java index 890e0ef8d51..3b1b0ab2ef3 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java @@ -6,6 +6,9 @@ package org.opensearch.sql.data.model; import inet.ipaddr.IPAddress; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -128,16 +131,22 @@ public static ExprValue fromObjectValue(Object o) { return stringValue((String) o); } else if (o instanceof Float) { return floatValue((Float) o); + } else if (o instanceof Date) { + return dateValue(((Date) o).toLocalDate()); } else if (o instanceof LocalDate) { return dateValue((LocalDate) o); + } else if (o instanceof Time) { + return timeValue(((Time) o).toLocalTime()); } else if (o instanceof LocalTime) { return timeValue((LocalTime) o); } else if (o instanceof Instant) { return timestampValue((Instant) o); - } else if (o instanceof TemporalAmount) { - return intervalValue((TemporalAmount) o); + } else if (o instanceof Timestamp) { + return timestampValue(((Timestamp) o).toInstant()); } else if (o instanceof LocalDateTime) { return timestampValue(((LocalDateTime) o).toInstant(ZoneOffset.UTC)); + } else if (o instanceof TemporalAmount) { + return intervalValue((TemporalAmount) o); } else { throw new ExpressionEvaluationException("unsupported object " + o.getClass()); } diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 456976f063d..d9de841e401 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -83,6 +83,13 @@ public void execute( return null; }); } catch (Exception e) { + boolean fallbackAllowed = true; + if (settings != null) { + fallbackAllowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED); + } + if (!fallbackAllowed) { + throw e; + } LOG.warn("Fallback to V2 query engine since got exception", e); executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index d7635de6102..921b3715620 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -102,7 +102,7 @@ public void testExtractDatePartWithTimeType() { // the previous year, and for late-December dates to be part of the first week of the next year. // For example, 2005-01-02 is part of the 53rd week of year 2004, while 2012-12-31 is part of // the first week of 2013 - if (now.getMonthValue() != 1 && now.getMonthValue() != 12) { + if (now.getMonthValue() > 2 && now.getMonthValue() != 12) { datePartWithTimeArgQuery("WEEK", datetimeInput, now.get(ALIGNED_WEEK_OF_YEAR)); } diff --git a/doctest/build.gradle b/doctest/build.gradle index 91d54c9cb24..96bb69231d7 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -100,8 +100,8 @@ task stopPrometheus(type: KillProcessTask) { if(getOSFamilyType() != "windows") { stopPrometheus.mustRunAfter startPrometheus startOpenSearch.dependsOn startPrometheus - stopOpenSearch.finalizedBy stopPrometheus - startOpenSearch.finalizedBy stopPrometheus +// stopOpenSearch.finalizedBy stopPrometheus +// startOpenSearch.finalizedBy stopPrometheus } doctest.dependsOn startOpenSearch doctest.finalizedBy stopOpenSearch diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 87d9f66d2a5..79fb637e631 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -157,11 +157,15 @@ configurations.all { resolutionStrategy.force "commons-logging:commons-logging:1.2" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" + resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" + resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-annotations:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.18.2" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.12.0" resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10" @@ -195,7 +199,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Needed for BWC tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteCsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteCsvFormatIT.java new file mode 100644 index 00000000000..fbe5a038d5b --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteCsvFormatIT.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import org.opensearch.sql.ppl.CsvFormatIT; + +public class CalciteCsvFormatIT extends CsvFormatIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java new file mode 100644 index 00000000000..ae03c68bc31 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import org.junit.Ignore; +import org.junit.Test; +import org.opensearch.sql.ppl.DataTypeIT; + +public class CalciteDataTypeIT extends DataTypeIT { + + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } + + @Override + @Test + @Ignore("ignore this class since IP type is unsupported in calcite engine") + public void test_nonnumeric_data_types() throws IOException { + super.test_nonnumeric_data_types(); + } + + @Override + @Test + @Ignore("ignore this class since the data type is unmatched in calcite engine") + public void test_numeric_data_types() throws IOException { + super.test_numeric_data_types(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java new file mode 100644 index 00000000000..8884fd45a41 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import org.junit.Ignore; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.FieldsCommandIT; + +public class CalciteFieldsCommandIT extends FieldsCommandIT { + + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } + + @Override + @Test + @Ignore("Calcite doesn't support metadata fields in fields yet") + public void testDelimitedMetadataFields() throws IOException { + super.testDelimitedMetadataFields(); + } + + @Override + @Test + @Ignore("Calcite doesn't support metadata fields in fields yet") + public void testMetadataFields() throws IOException { + super.testMetadataFields(); + } + + @Override + @Test + @Ignore("https://github.com/opensearch-project/sql/issues/3341") + public void testSelectDateTypeField() throws IOException { + super.testSelectDateTypeField(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteHeadCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteHeadCommandIT.java new file mode 100644 index 00000000000..48a3e6b66a3 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteHeadCommandIT.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import org.opensearch.sql.ppl.HeadCommandIT; + +public class CalciteHeadCommandIT extends HeadCommandIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java new file mode 100644 index 00000000000..8bcd801c121 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import org.junit.Ignore; +import org.opensearch.sql.ppl.LikeQueryIT; + +@Ignore("CalciteLikeQueryIT is not supported in OpenSearch yet") +public class CalciteLikeQueryIT extends LikeQueryIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPluginIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPluginIT.java new file mode 100644 index 00000000000..447b738bc17 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPluginIT.java @@ -0,0 +1,12 @@ +package org.opensearch.sql.calcite.remote; + +import org.opensearch.sql.ppl.PPLPluginIT; + +public class CalcitePPLPluginIT extends PPLPluginIT { + @Override + public void init() throws Exception { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java new file mode 100644 index 00000000000..1fb0acfeda1 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import org.junit.jupiter.api.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.QueryAnalysisIT; + +import java.io.IOException; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; + +public class CalciteQueryAnalysisIT extends QueryAnalysisIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } + + @Override + @Test + public void nonexistentFieldShouldFailSemanticCheck() { + String query = String.format("search source=%s | fields name", TEST_INDEX_ACCOUNT); + try { + executeQuery(query); + fail("Expected to throw Exception, but none was thrown for query: " + query); + } catch (ResponseException e) { + String errorMsg = e.getMessage(); + assertTrue(errorMsg.contains("IllegalArgumentException")); + assertTrue(errorMsg.contains("field [name] not found")); + } catch (IOException e) { + throw new IllegalStateException("Unexpected exception raised for query: " + query); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java new file mode 100644 index 00000000000..4751c98a3fa --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import org.junit.Ignore; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.SearchCommandIT; + +import java.io.IOException; + +public class CalciteSearchCommandIT extends SearchCommandIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } + + @Override + @Test + @Ignore("https://github.com/opensearch-project/sql/issues/3341") + public void testSearchCommandWithoutSearchKeyword() throws IOException { + super.testSearchCommandWithoutSearchKeyword(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java new file mode 100644 index 00000000000..78ed38073a7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import org.junit.Ignore; +import org.opensearch.sql.ppl.SortCommandIT; + +/** + * TODO there seems a bug in Calcite planner with sort. Fix {@link + * org.opensearch.sql.calcite.standalone.CalcitePPLSortIT} first. then enable this IT and remove + * this java doc. + */ +@Ignore +public class CalciteSortCommandIT extends SortCommandIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java new file mode 100644 index 00000000000..0a03c07db3a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import org.junit.Ignore; +import org.opensearch.sql.ppl.StatsCommandIT; + +import java.io.IOException; + +//TODO +@Ignore("Not all agg functions are supported in Calcite now") +public class CalciteStatsCommandIT extends StatsCommandIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnsupportedCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnsupportedCommandIT.java new file mode 100644 index 00000000000..a0a09776d16 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnsupportedCommandIT.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; + +import java.io.IOException; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteUnsupportedCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.BANK); + } + + @Test + public void test_match_function() throws IOException { + failWithMessage( + String.format( + "source=%s | where match(firstname, 'Hattie') | fields firstname", TEST_INDEX_BANK), + "Unsupported operator: match"); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java new file mode 100644 index 00000000000..2905a17a4fa --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import org.junit.Ignore; +import org.opensearch.sql.ppl.WhereCommandIT; + +import java.io.IOException; + +@Ignore("Not all boolean functions are supported in Calcite now") +public class CalciteWhereCommandIT extends WhereCommandIT { + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java similarity index 98% rename from integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLAggregationIT.java rename to integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java index ac3d610ba20..85055b06980 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java @@ -3,15 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.calcite; +package org.opensearch.sql.calcite.standalone; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import java.io.IOException; +import org.junit.Ignore; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; -public class CalcitePPLAggregationIT extends CalcitePPLIntegTestCase { +public class CalcitePPLAggregationIT extends CalcitePPLTestCase { @Override public void init() throws IOException { @@ -268,7 +269,7 @@ public void testAvgBySpanAndFields() { } // TODO fallback to V2 because missing conversion LogicalAggregate[convention: NONE -> ENUMERABLE] - @Test + @Ignore public void testCountDistinct() { String actual = execute( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java similarity index 78% rename from integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLBasicIT.java rename to integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java index 6bb37b5b0d1..2a1569cf035 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.calcite; +package org.opensearch.sql.calcite.standalone; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; @@ -12,7 +12,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; -public class CalcitePPLBasicIT extends CalcitePPLIntegTestCase { +public class CalcitePPLBasicIT extends CalcitePPLTestCase { @Override public void init() throws IOException { @@ -35,6 +35,76 @@ public void testInvalidTable() { () -> execute("source=unknown")); } + @Test + public void testSourceQuery() { + String actual = execute("source=test"); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"name\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"hello\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"world\",\n" + + " 30\n" + + " ]\n" + + " ],\n" + + " \"total\": 2,\n" + + " \"size\": 2\n" + + "}", + actual); + } + + @Test + public void testMultipleSourceQuery() { + String actual = execute("source=test, test"); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"name\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"hello\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"world\",\n" + + " 30\n" + + " ],\n" + + " [\n" + + " \"hello\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"world\",\n" + + " 30\n" + + " ]\n" + + " ],\n" + + " \"total\": 4,\n" + + " \"size\": 4\n" + + "}", + actual); + } + @Test public void testSourceFieldQuery() { String actual = execute("source=test | fields name"); @@ -145,6 +215,8 @@ public void testFilterQuery3() { actual); } + // TODO fail after merged https://github.com/opensearch-project/sql/pull/3327 + @Ignore @Test public void testFilterQueryWithOr() { String actual = @@ -181,6 +253,8 @@ public void testFilterQueryWithOr() { actual); } + // TODO fail after merged https://github.com/opensearch-project/sql/pull/3327 + @Ignore @Test public void testFilterQueryWithOr2() { String actual = @@ -220,7 +294,7 @@ public void testFilterQueryWithOr2() { @Test public void testQueryMinusFields() { String actual = - execute(String.format("source=%s | fields - firstname, lastname", TEST_INDEX_BANK)); + execute(String.format("source=%s | fields - firstname, lastname, birthdate", TEST_INDEX_BANK)); assertEquals( "{\n" + " \"schema\": [\n" @@ -233,10 +307,6 @@ public void testQueryMinusFields() { + " \"type\": \"string\"\n" + " },\n" + " {\n" - + " \"name\": \"birthdate\",\n" - + " \"type\": \"timestamp\"\n" - + " },\n" - + " {\n" + " \"name\": \"gender\",\n" + " \"type\": \"string\"\n" + " },\n" @@ -273,7 +343,6 @@ public void testQueryMinusFields() { + " [\n" + " 1,\n" + " \"880 Holmes Lane\",\n" - + " \"2017-10-23 00:00:00\",\n" + " \"M\",\n" + " \"Brogan\",\n" + " 39225,\n" @@ -286,7 +355,6 @@ public void testQueryMinusFields() { + " [\n" + " 6,\n" + " \"671 Bristol Street\",\n" - + " \"2017-11-20 00:00:00\",\n" + " \"M\",\n" + " \"Dante\",\n" + " 5686,\n" @@ -299,7 +367,6 @@ public void testQueryMinusFields() { + " [\n" + " 13,\n" + " \"789 Madison Street\",\n" - + " \"2018-06-23 00:00:00\",\n" + " \"F\",\n" + " \"Nogal\",\n" + " 32838,\n" @@ -312,7 +379,6 @@ public void testQueryMinusFields() { + " [\n" + " 18,\n" + " \"467 Hutchinson Court\",\n" - + " \"2018-11-13 23:33:20\",\n" + " \"M\",\n" + " \"Orick\",\n" + " 4180,\n" @@ -325,7 +391,6 @@ public void testQueryMinusFields() { + " [\n" + " 20,\n" + " \"282 Kings Place\",\n" - + " \"2018-06-27 00:00:00\",\n" + " \"M\",\n" + " \"Ribera\",\n" + " 16418,\n" @@ -338,7 +403,6 @@ public void testQueryMinusFields() { + " [\n" + " 25,\n" + " \"171 Putnam Avenue\",\n" - + " \"2018-08-19 00:00:00\",\n" + " \"F\",\n" + " \"Nicholson\",\n" + " 40540,\n" @@ -351,7 +415,6 @@ public void testQueryMinusFields() { + " [\n" + " 32,\n" + " \"702 Quentin Street\",\n" - + " \"2018-08-11 00:00:00\",\n" + " \"F\",\n" + " \"Veguita\",\n" + " 48086,\n" @@ -477,238 +540,83 @@ public void testFieldsPlusThenMinus() { actual); } + // TODO fail after merged https://github.com/opensearch-project/sql/pull/3327 + @Ignore @Test - public void testSort() { + public void testMultipleTables() { String actual = execute( - String.format( - "source=%s | fields + firstname, gender, account_number | sort - account_number", - TEST_INDEX_BANK)); + String.format("source=%s, %s | stats count() as c", TEST_INDEX_BANK, TEST_INDEX_BANK)); assertEquals( "{\n" + " \"schema\": [\n" + " {\n" - + " \"name\": \"firstname\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"gender\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"account_number\",\n" + + " \"name\": \"c\",\n" + " \"type\": \"long\"\n" + " }\n" + " ],\n" + " \"datarows\": [\n" + " [\n" - + " \"Dillard\",\n" - + " \"F\",\n" - + " 32\n" - + " ],\n" - + " [\n" - + " \"Virginia\",\n" - + " \"F\",\n" - + " 25\n" - + " ],\n" - + " [\n" - + " \"Elinor\",\n" - + " \"M\",\n" - + " 20\n" - + " ],\n" - + " [\n" - + " \"Dale\",\n" - + " \"M\",\n" - + " 18\n" - + " ],\n" - + " [\n" - + " \"Nanette\",\n" - + " \"F\",\n" - + " 13\n" - + " ],\n" - + " [\n" - + " \"Hattie\",\n" - + " \"M\",\n" - + " 6\n" - + " ],\n" - + " [\n" - + " \"Amber JOHnny\",\n" - + " \"M\",\n" - + " 1\n" + + " 14\n" + " ]\n" + " ],\n" - + " \"total\": 7,\n" - + " \"size\": 7\n" + + " \"total\": 1,\n" + + " \"size\": 1\n" + "}", actual); } + // TODO fail after merged https://github.com/opensearch-project/sql/pull/3327 + @Ignore @Test - public void testSortTwoFields() { + public void testMultipleTablesAndFilters() { String actual = execute( String.format( - "source=%s | fields + firstname, gender, account_number | sort + gender, -" - + " account_number", - TEST_INDEX_BANK)); + "source=%s, %s gender = 'F' | stats count() as c", + TEST_INDEX_BANK, TEST_INDEX_BANK)); assertEquals( "{\n" + " \"schema\": [\n" + " {\n" - + " \"name\": \"firstname\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"gender\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"account_number\",\n" + + " \"name\": \"c\",\n" + " \"type\": \"long\"\n" + " }\n" + " ],\n" + " \"datarows\": [\n" + " [\n" - + " \"Dillard\",\n" - + " \"F\",\n" - + " 32\n" - + " ],\n" - + " [\n" - + " \"Virginia\",\n" - + " \"F\",\n" - + " 25\n" - + " ],\n" - + " [\n" - + " \"Nanette\",\n" - + " \"F\",\n" - + " 13\n" - + " ],\n" - + " [\n" - + " \"Elinor\",\n" - + " \"M\",\n" - + " 20\n" - + " ],\n" - + " [\n" - + " \"Dale\",\n" - + " \"M\",\n" - + " 18\n" - + " ],\n" - + " [\n" - + " \"Hattie\",\n" - + " \"M\",\n" + " 6\n" - + " ],\n" - + " [\n" - + " \"Amber JOHnny\",\n" - + " \"M\",\n" - + " 1\n" + " ]\n" + " ],\n" - + " \"total\": 7,\n" - + " \"size\": 7\n" + + " \"total\": 1,\n" + + " \"size\": 1\n" + "}", actual); } - @Test - public void testSortWithDescAndLimit() { - String actual = - execute( - String.format( - "source=%s | fields + firstname, gender, account_number | sort + gender, -" - + " account_number | head 5", - TEST_INDEX_BANK)); - assertEquals( - "{\n" - + " \"schema\": [\n" - + " {\n" - + " \"name\": \"firstname\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"gender\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"account_number\",\n" - + " \"type\": \"long\"\n" - + " }\n" - + " ],\n" - + " \"datarows\": [\n" - + " [\n" - + " \"Dillard\",\n" - + " \"F\",\n" - + " 32\n" - + " ],\n" - + " [\n" - + " \"Virginia\",\n" - + " \"F\",\n" - + " 25\n" - + " ],\n" - + " [\n" - + " \"Nanette\",\n" - + " \"F\",\n" - + " 13\n" - + " ],\n" - + " [\n" - + " \"Elinor\",\n" - + " \"M\",\n" - + " 20\n" - + " ],\n" - + " [\n" - + " \"Dale\",\n" - + " \"M\",\n" - + " 18\n" - + " ]\n" - + " ],\n" - + " \"total\": 5,\n" - + " \"size\": 5\n" - + "}", - actual); + @Ignore // https://github.com/opensearch-project/sql/issues/3341 + public void testSelectDateTypeField() { + String actual = execute(String.format("source=%s | fields birthdate", TEST_INDEX_BANK)); } @Test - public void testMultipleTables() { - String actual = - execute( - String.format("source=%s, %s | stats count() as c", TEST_INDEX_BANK, TEST_INDEX_BANK)); - assertEquals( - "{\n" - + " \"schema\": [\n" - + " {\n" - + " \"name\": \"c\",\n" - + " \"type\": \"long\"\n" - + " }\n" - + " ],\n" - + " \"datarows\": [\n" - + " [\n" - + " 14\n" - + " ]\n" - + " ],\n" - + " \"total\": 1,\n" - + " \"size\": 1\n" - + "}", - actual); - } + public void testAllFieldsInTable() throws IOException { + Request request = new Request("PUT", "/a/_doc/1?refresh=true"); + request.setJsonEntity("{\"name\": \"hello\"}"); + client().performRequest(request); - @Test - public void testMultipleTablesAndFilters() { - String actual = - execute( - String.format( - "source=%s, %s gender = 'F' | stats count() as c", - TEST_INDEX_BANK, TEST_INDEX_BANK)); + String actual = execute("source=a | fields name"); assertEquals( "{\n" + " \"schema\": [\n" + " {\n" - + " \"name\": \"c\",\n" - + " \"type\": \"long\"\n" + + " \"name\": \"name\",\n" + + " \"type\": \"string\"\n" + " }\n" + " ],\n" + " \"datarows\": [\n" + " [\n" - + " 6\n" + + " \"hello\"\n" + " ]\n" + " ],\n" + " \"total\": 1,\n" diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java new file mode 100644 index 00000000000..ad2f9d9686b --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java @@ -0,0 +1,552 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; + +import java.io.IOException; +import org.junit.jupiter.api.Test; + +/** testSortXXAndXX could fail. TODO Remove this @Ignore when the issue fixed. */ +// @Ignore +public class CalcitePPLSortIT extends CalcitePPLTestCase { + + @Override + public void init() throws IOException { + super.init(); + + loadIndex(Index.BANK); + } + + @Test + public void testFieldsAndSort1() { + String actual = + execute( + String.format( + "source=%s | fields + firstname, gender, account_number | sort - account_number", + TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"gender\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"account_number\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"Dillard\",\n" + + " \"F\",\n" + + " 32\n" + + " ],\n" + + " [\n" + + " \"Virginia\",\n" + + " \"F\",\n" + + " 25\n" + + " ],\n" + + " [\n" + + " \"Elinor\",\n" + + " \"M\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Dale\",\n" + + " \"M\",\n" + + " 18\n" + + " ],\n" + + " [\n" + + " \"Nanette\",\n" + + " \"F\",\n" + + " 13\n" + + " ],\n" + + " [\n" + + " \"Hattie\",\n" + + " \"M\",\n" + + " 6\n" + + " ],\n" + + " [\n" + + " \"Amber JOHnny\",\n" + + " \"M\",\n" + + " 1\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}", + actual); + } + + @Test + public void testFieldsAndSort2() { + String actual = execute(String.format("source=%s | fields age | sort - age", TEST_INDEX_BANK)); + String expected = + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " 39\n" + + " ],\n" + + " [\n" + + " 36\n" + + " ],\n" + + " [\n" + + " 36\n" + + " ],\n" + + " [\n" + + " 34\n" + + " ],\n" + + " [\n" + + " 33\n" + + " ],\n" + + " [\n" + + " 32\n" + + " ],\n" + + " [\n" + + " 28\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}"; + assertEquals(expected, actual); + } + + @Test + public void testFieldsAndSortTwoFields() { + String actual = + execute( + String.format( + "source=%s | fields + firstname, gender, account_number | sort + gender, -" + + " account_number", + TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"gender\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"account_number\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"Dillard\",\n" + + " \"F\",\n" + + " 32\n" + + " ],\n" + + " [\n" + + " \"Virginia\",\n" + + " \"F\",\n" + + " 25\n" + + " ],\n" + + " [\n" + + " \"Nanette\",\n" + + " \"F\",\n" + + " 13\n" + + " ],\n" + + " [\n" + + " \"Elinor\",\n" + + " \"M\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Dale\",\n" + + " \"M\",\n" + + " 18\n" + + " ],\n" + + " [\n" + + " \"Hattie\",\n" + + " \"M\",\n" + + " 6\n" + + " ],\n" + + " [\n" + + " \"Amber JOHnny\",\n" + + " \"M\",\n" + + " 1\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}", + actual); + } + + @Test + public void testFieldsAndSortWithDescAndLimit() { + String actual = + execute( + String.format( + "source=%s | fields + firstname, gender, account_number | sort + gender, -" + + " account_number | head 5", + TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"gender\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"account_number\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"Dillard\",\n" + + " \"F\",\n" + + " 32\n" + + " ],\n" + + " [\n" + + " \"Virginia\",\n" + + " \"F\",\n" + + " 25\n" + + " ],\n" + + " [\n" + + " \"Nanette\",\n" + + " \"F\",\n" + + " 13\n" + + " ],\n" + + " [\n" + + " \"Elinor\",\n" + + " \"M\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Dale\",\n" + + " \"M\",\n" + + " 18\n" + + " ]\n" + + " ],\n" + + " \"total\": 5,\n" + + " \"size\": 5\n" + + "}", + actual); + } + + @Test + public void testSortAccountAndFieldsAccount() { + String actual = + execute( + String.format( + "source=%s | sort - account_number | fields account_number", TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"account_number\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " 32\n" + + " ],\n" + + " [\n" + + " 25\n" + + " ],\n" + + " [\n" + + " 20\n" + + " ],\n" + + " [\n" + + " 18\n" + + " ],\n" + + " [\n" + + " 13\n" + + " ],\n" + + " [\n" + + " 6\n" + + " ],\n" + + " [\n" + + " 1\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}", + actual); + } + + @Test + public void testSortAccountAndFieldsNameAccount() { + String actual = + execute( + String.format( + "source=%s | sort - account_number | fields firstname, account_number", + TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"account_number\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"Dillard\",\n" + + " 32\n" + + " ],\n" + + " [\n" + + " \"Virginia\",\n" + + " 25\n" + + " ],\n" + + " [\n" + + " \"Elinor\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Dale\",\n" + + " 18\n" + + " ],\n" + + " [\n" + + " \"Nanette\",\n" + + " 13\n" + + " ],\n" + + " [\n" + + " \"Hattie\",\n" + + " 6\n" + + " ],\n" + + " [\n" + + " \"Amber JOHnny\",\n" + + " 1\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}", + actual); + } + + @Test + public void testSortAccountAndFieldsAccountName() { + String actual = + execute( + String.format( + "source=%s | sort - account_number | fields account_number, firstname", + TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"account_number\",\n" + + " \"type\": \"long\"\n" + + " },\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " 32,\n" + + " \"Dillard\"\n" + + " ],\n" + + " [\n" + + " 25,\n" + + " \"Virginia\"\n" + + " ],\n" + + " [\n" + + " 20,\n" + + " \"Elinor\"\n" + + " ],\n" + + " [\n" + + " 18,\n" + + " \"Dale\"\n" + + " ],\n" + + " [\n" + + " 13,\n" + + " \"Nanette\"\n" + + " ],\n" + + " [\n" + + " 6,\n" + + " \"Hattie\"\n" + + " ],\n" + + " [\n" + + " 1,\n" + + " \"Amber JOHnny\"\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}", + actual); + } + + @Test + public void testSortAgeAndFieldsAge() { + String actual = execute(String.format("source=%s | sort - age | fields age", TEST_INDEX_BANK)); + String expected = + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " 39\n" + + " ],\n" + + " [\n" + + " 36\n" + + " ],\n" + + " [\n" + + " 36\n" + + " ],\n" + + " [\n" + + " 34\n" + + " ],\n" + + " [\n" + + " 33\n" + + " ],\n" + + " [\n" + + " 32\n" + + " ],\n" + + " [\n" + + " 28\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}"; + assertEquals(expected, actual); + } + + @Test + public void testSortAgeAndFieldsNameAge() { + String actual = + execute(String.format("source=%s | sort - age | fields firstname, age", TEST_INDEX_BANK)); + String expected = + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"Virginia\",\n" + + " 39\n" + + " ],\n" + + " [\n" + + " \"Hattie\",\n" + + " 36\n" + + " ],\n" + + " [\n" + + " \"Elinor\",\n" + + " 36\n" + + " ],\n" + + " [\n" + + " \"Dillard\",\n" + + " 34\n" + + " ],\n" + + " [\n" + + " \"Dale\",\n" + + " 33\n" + + " ],\n" + + " [\n" + + " \"Amber JOHnny\",\n" + + " 32\n" + + " ],\n" + + " [\n" + + " \"Nanette\",\n" + + " 28\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}"; + assertEquals(expected, actual); + } + + @Test + public void testSortAgeNameAndFieldsNameAge() { + String actual = + execute( + String.format( + "source=%s | sort - age, - firstname | fields firstname, age", TEST_INDEX_BANK)); + String expected = + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"Virginia\",\n" + + " 39\n" + + " ],\n" + + " [\n" + + " \"Hattie\",\n" + + " 36\n" + + " ],\n" + + " [\n" + + " \"Elinor\",\n" + + " 36\n" + + " ],\n" + + " [\n" + + " \"Dillard\",\n" + + " 34\n" + + " ],\n" + + " [\n" + + " \"Dale\",\n" + + " 33\n" + + " ],\n" + + " [\n" + + " \"Amber JOHnny\",\n" + + " 32\n" + + " ],\n" + + " [\n" + + " \"Nanette\",\n" + + " 28\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}"; + assertEquals(expected, actual); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLTestCase.java similarity index 93% rename from integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLIntegTestCase.java rename to integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLTestCase.java index 22d196cfc90..7a3c78e929b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalcitePPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLTestCase.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.calcite; +package org.opensearch.sql.calcite.standalone; import static org.opensearch.sql.datasource.model.DataSourceMetadata.defaultOpenSearchDataSourceMetadata; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; @@ -26,6 +26,7 @@ import org.opensearch.common.inject.Singleton; import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.analysis.ExpressionAnalyzer; +import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.DataSourceService; @@ -63,13 +64,16 @@ import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.util.ExecuteOnCallerThreadQueryManager; -public abstract class CalcitePPLIntegTestCase extends PPLIntegTestCase { +/** + * This abstract test case provide a standalone env to run PPL query, IT extends this class could + * debug the service side execution of PPL in IDE. + */ +public abstract class CalcitePPLTestCase extends PPLIntegTestCase { protected PPLService pplService; @Override public void init() throws IOException { - RestHighLevelClient restClient = - new CalcitePPLIntegTestCase.InternalRestHighLevelClient(client()); + RestHighLevelClient restClient = new CalcitePPLTestCase.InternalRestHighLevelClient(client()); OpenSearchClient client = new OpenSearchRestClient(restClient); DataSourceService dataSourceService = new DataSourceServiceImpl( @@ -82,8 +86,8 @@ public void init() throws IOException { ModulesBuilder modules = new ModulesBuilder(); modules.add( - new CalcitePPLIntegTestCase.StandaloneModule( - new CalcitePPLIntegTestCase.InternalRestHighLevelClient(client()), + new CalcitePPLTestCase.StandaloneModule( + new CalcitePPLTestCase.InternalRestHighLevelClient(client()), defaultSettings(), dataSourceService)); Injector injector = modules.createInjector(); @@ -98,6 +102,7 @@ private Settings defaultSettings() { .put(Key.SQL_PAGINATION_API_SEARCH_AFTER, true) .put(Key.FIELD_TYPE_TOLERANCE, true) .put(Key.CALCITE_ENGINE_ENABLED, true) + .put(Key.CALCITE_FALLBACK_ALLOWED, false) .build(); @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 459788021da..bc1863949aa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -17,6 +17,7 @@ import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; /** OpenSearch Rest integration test base for PPL testing. */ @@ -52,6 +53,14 @@ protected String executeCsvQuery(String query) throws IOException { return executeCsvQuery(query, true); } + protected void failWithMessage(String query, String message) { + try { + client().performRequest(buildRequest(query, QUERY_API_ENDPOINT)); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains(message)); + } + } + protected Request buildRequest(String query, String endpoint) { Request request = new Request("POST", endpoint); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); @@ -102,4 +111,28 @@ private JSONObject jsonify(String text) { throw new IllegalStateException(String.format("Failed to transform %s to JSON format", text)); } } + + protected static void enableCalcite() throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "persistent", Settings.Key.CALCITE_ENGINE_ENABLED.getKeyValue(), "true")); + } + + protected static void disableCalcite() throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "persistent", Settings.Key.CALCITE_ENGINE_ENABLED.getKeyValue(), "false")); + } + + protected static void allowCalciteFallback() throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "persistent", Settings.Key.CALCITE_FALLBACK_ALLOWED.getKeyValue(), "true")); + } + + protected static void disallowCalciteFallback() throws IOException { + updateClusterSettings( + new SQLIntegTestCase.ClusterSetting( + "persistent", Settings.Key.CALCITE_FALLBACK_ALLOWED.getKeyValue(), "false")); + } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java index e951c849617..19b5a605ca7 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import org.apache.lucene.search.TotalHits; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.document.DocumentField; import org.opensearch.search.SearchHit; @@ -37,8 +38,7 @@ public class SearchResult { public SearchResult(SearchResponse resp) { SearchHits hits = resp.getHits(); - this.total = - Optional.ofNullable(hits.getTotalHits()).map(totalHits -> totalHits.value).orElse(0L); + this.total = Optional.ofNullable(hits.getTotalHits()).map(TotalHits::value).orElse(0L); results = new ArrayList<>(hits.getHits().length); for (SearchHit searchHit : hits.getHits()) { if (searchHit.getSourceAsMap() != null) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index 33448298596..d533e003119 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -136,7 +136,7 @@ private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction protected boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { return queryAction.getSqlRequest().fetchSize() != 0 - && Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value + && Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value() >= queryAction.getSqlRequest().fetchSize(); } else { return !Strings.isNullOrEmpty(searchResponse.getScrollId()); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index 84b7c00857f..e0f6c189069 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -26,6 +26,7 @@ import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.TotalHits; import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; @@ -541,7 +542,7 @@ private void extractData() { this.rows = populateRows(searchHits); this.size = rows.size(); this.internalTotalHits = - Optional.ofNullable(searchHits.getTotalHits()).map(th -> th.value).orElse(0L); + Optional.ofNullable(searchHits.getTotalHits()).map(TotalHits::value).orElse(0L); // size may be greater than totalHits after nested rows be flatten this.totalHits = Math.max(size, internalTotalHits); } else if (queryResult instanceof Aggregations) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java index 70e7118ad5f..028861aa47c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java @@ -13,6 +13,7 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; @@ -62,7 +63,10 @@ public static String hitsAsStringResult(SearchHits results, MetaSearchResult met Object[] searchHits; searchHits = new Object - [Optional.ofNullable(results.getTotalHits()).map(th -> th.value).orElse(0L).intValue()]; + [Optional.ofNullable(results.getTotalHits()) + .map(TotalHits::value) + .orElse(0L) + .intValue()]; int i = 0; for (SearchHit hit : results) { HashMap value = new HashMap<>(); @@ -76,10 +80,10 @@ public static String hitsAsStringResult(SearchHits results, MetaSearchResult met hits.put( "total", ImmutableMap.of( - "value", Optional.ofNullable(results.getTotalHits()).map(th -> th.value).orElse(0L), + "value", Optional.ofNullable(results.getTotalHits()).map(TotalHits::value).orElse(0L), "relation", Optional.ofNullable(results.getTotalHits()) - .map(th -> th.relation) + .map(TotalHits::relation) .orElse(Relation.EQUAL_TO))); hits.put("max_score", results.getMaxScore()); hits.put("hits", searchHits); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index f4e4347e067..36335db9969 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -305,7 +305,7 @@ private FetchWithScrollResponse firstFetch(TableInJoinRequestBuilder tableReques // scroll request with max. responseWithHits = getResponseWithHits(tableRequest, MAX_RESULTS_ON_ONE_FETCH, null); if (responseWithHits.getHits().getTotalHits() != null - && responseWithHits.getHits().getTotalHits().value < MAX_RESULTS_ON_ONE_FETCH) { + && responseWithHits.getHits().getTotalHits().value() < MAX_RESULTS_ON_ONE_FETCH) { needScrollForFirstTable = true; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 43b6f3b9cf4..26debff95dd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -18,6 +18,8 @@ import lombok.RequiredArgsConstructor; import org.apache.calcite.rel.RelNode; import org.apache.calcite.tools.RelRunners; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprTupleValue; @@ -30,13 +32,14 @@ import org.opensearch.sql.executor.pagination.PlanSerializer; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; -import org.opensearch.sql.opensearch.util.JdbcUtil; +import org.opensearch.sql.opensearch.util.JdbcOpenSearchDataTypeConvertor; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.TableScanOperator; /** OpenSearch execution engine implementation. */ @RequiredArgsConstructor public class OpenSearchExecutionEngine implements ExecutionEngine { + private static final Logger LOG = LogManager.getLogger(); private final OpenSearchClient client; @@ -134,7 +137,8 @@ private void buildResultSet(ResultSet resultSet, ResponseListener for (int i = 1; i <= columnCount; i++) { String columnName = metaData.getColumnName(i); int sqlType = metaData.getColumnType(i); - ExprValue exprValue = JdbcUtil.getExprValueFromSqlType(resultSet, i, sqlType); + ExprValue exprValue = + JdbcOpenSearchDataTypeConvertor.getExprValueFromSqlType(resultSet, i, sqlType); row.put(columnName, exprValue); } values.add(ExprTupleValue.fromExprValueMap(row)); @@ -144,7 +148,7 @@ private void buildResultSet(ResultSet resultSet, ResponseListener for (int i = 1; i <= columnCount; ++i) { String columnName = metaData.getColumnName(i); int sqlType = metaData.getColumnType(i); - ExprType exprType = JdbcUtil.getExprTypeFromSqlType(sqlType); + ExprType exprType = JdbcOpenSearchDataTypeConvertor.getExprTypeFromSqlType(sqlType); columns.add(new Column(columnName, null, exprType)); } Schema schema = new Schema(columns); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java new file mode 100644 index 00000000000..69c3653d3bb --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.physical; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableProject; +import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.logical.LogicalProject; +import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalTableScan; +import org.opensearch.sql.opensearch.storage.scan.CalciteOpenSearchIndexScan; + +/** + * Rule to convert a {@link LogicalProject} to an {@link EnumerableProject}. You may provide a + * custom config to convert other nodes that extend {@link Project}. + * + * @see EnumerableRules#ENUMERABLE_PROJECT_RULE + */ +public class EnumerableIndexScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalTableScan.class, + s -> s.getOsIndex() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableIndexScanRule") + .withRuleFactory(EnumerableIndexScanRule::new); + + /** Creates an EnumerableProjectRule. */ + protected EnumerableIndexScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalTableScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalTableScan scan = (CalciteLogicalTableScan) rel; + return new CalciteOpenSearchIndexScan(scan.getCluster(), scan.getTable(), scan.getOsIndex()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 91c0c9e7352..53bf6536c96 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -92,6 +92,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting CALCITE_FALLBACK_ALLOWED_SETTING = + Setting.boolSetting( + Key.CALCITE_FALLBACK_ALLOWED.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting QUERY_MEMORY_LIMIT_SETTING = new Setting<>( Key.QUERY_MEMORY_LIMIT.getKeyValue(), @@ -289,6 +296,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.CALCITE_ENGINE_ENABLED, CALCITE_ENGINE_ENABLED_SETTING, new Updater(Key.CALCITE_ENGINE_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.CALCITE_FALLBACK_ALLOWED, + CALCITE_FALLBACK_ALLOWED_SETTING, + new Updater(Key.CALCITE_FALLBACK_ALLOWED)); register( settingBuilder, clusterSettings, @@ -464,6 +477,7 @@ public static List> pluginSettings() { .add(SQL_PAGINATION_API_SEARCH_AFTER_SETTING) .add(PPL_ENABLED_SETTING) .add(CALCITE_ENGINE_ENABLED_SETTING) + .add(CALCITE_FALLBACK_ALLOWED_SETTING) .add(QUERY_MEMORY_LIMIT_SETTING) .add(QUERY_SIZE_LIMIT_SETTING) .add(METRICS_ROLLING_WINDOW_SETTING) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index b7b98116fed..19212a662d5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -33,6 +33,7 @@ import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalTableScan; import org.opensearch.sql.opensearch.storage.scan.CalciteOpenSearchIndexScan; import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexEnumerator; import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexScan; @@ -93,7 +94,8 @@ public OpenSearchIndex(OpenSearchClient client, Settings settings, String indexN @Override public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { final RelOptCluster cluster = context.getCluster(); - return new CalciteOpenSearchIndexScan(cluster, relOptTable, this); +// return new CalciteOpenSearchIndexScan(cluster, relOptTable, this); + return new CalciteLogicalTableScan(cluster, relOptTable, this); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalTableScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalTableScan.java new file mode 100644 index 00000000000..30b2193d927 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalTableScan.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.Getter; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.hint.RelHint; +import org.opensearch.sql.opensearch.planner.physical.EnumerableIndexScanRule; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; + +@Getter +public class CalciteLogicalTableScan extends TableScan { + private final OpenSearchIndex osIndex; + + protected CalciteLogicalTableScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + OpenSearchIndex osIndex) { + super(cluster, traitSet, hints, table); + this.osIndex = osIndex; + } + + public CalciteLogicalTableScan( + RelOptCluster cluster, RelOptTable table, OpenSearchIndex osIndex) { + this(cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, osIndex); + } + + @Override + public void register(RelOptPlanner planner) { + super.register(planner); + planner.addRule(EnumerableIndexScanRule.DEFAULT_CONFIG.toRule()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java index 20fb52c1125..84530925bc6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java @@ -124,7 +124,7 @@ public boolean pushDownFilter(Filter filter) { QueryBuilder filterBuilder = PredicateAnalyzer.analyze(filter.getCondition(), schema); requestBuilder.pushDownFilter(filterBuilder); // TODO: handle the case where condition contains a score function - return true; + return false; } catch (ExpressionNotAnalyzableException | PredicateAnalyzerException e) { LOG.warn("Cannot analyze the filter condition {}", filter.getCondition(), e); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java index 161badffb6e..c87ea0df5f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java @@ -54,11 +54,13 @@ private void fetchNextBatch() { OpenSearchResponse response = client.search(request); if (!response.isEmpty()) { iterator = response.iterator(); + } else if (iterator == null){ + iterator = Collections.emptyIterator(); } } @Override - public Object[] current() { + public Object current() { Object[] p = fields.stream().map(k -> current.tupleValue().get(k).value()).toArray(); return p; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java similarity index 95% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcUtil.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java index 8abd884049a..d8d3ae9ecac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java @@ -16,8 +16,9 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +/** This class is used to convert the data type from JDBC to OpenSearch data type. */ @UtilityClass -public class JdbcUtil { +public class JdbcOpenSearchDataTypeConvertor { private static final Logger LOG = LogManager.getLogger(); public static ExprType getExprTypeFromSqlType(int sqlType) { diff --git a/plugin/build.gradle b/plugin/build.gradle index 9df3d3dd483..b0b7a5a8053 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -170,7 +170,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${versions.mockito}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" } test { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java index 9ee4f81b9eb..461b1bc265d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBasicTest.java @@ -253,6 +253,7 @@ public void testMultipleTables() { + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); + verifyResultCount(root, 28); } @Test From 64177ae822d852bb2c7f6defd2ffb81ed9054e62 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Fri, 21 Feb 2025 18:10:18 +0800 Subject: [PATCH 2/5] make local work Signed-off-by: Lantao Jin --- async-query-core/build.gradle | 16 ++++++++-------- .../spark/execution/statestore/StateStore.java | 2 +- .../spark/flint/OpenSearchFlintIndexClient.java | 2 +- build.gradle | 12 ++++++------ doctest/build.gradle | 4 ++-- integ-test/build.gradle | 14 +++++++------- .../sql/legacy/domain/SearchResult.java | 4 ++-- .../format/PrettyFormatRestExecutor.java | 2 +- .../legacy/executor/format/SelectResultSet.java | 3 +-- .../sql/legacy/executor/join/ElasticUtils.java | 10 +++------- .../join/NestedLoopsElasticExecutor.java | 2 +- .../storage/scan/CalciteOpenSearchIndexScan.java | 2 +- plugin/build.gradle | 2 +- 13 files changed, 35 insertions(+), 40 deletions(-) diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index 2c0f8292aab..37bf6748c9d 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -141,11 +141,11 @@ jacocoTestCoverageVerification { } check.dependsOn jacocoTestCoverageVerification -//shadowJar { -// archiveBaseName.set('async-query-core') -// archiveVersion.set('1.0.0') // Set the desired version -// archiveClassifier.set('all') -// -// from sourceSets.main.output -// configurations = [project.configurations.runtimeClasspath] -//} \ No newline at end of file +shadowJar { + archiveBaseName.set('async-query-core') + archiveVersion.set('1.0.0') // Set the desired version + archiveClassifier.set('all') + + from sourceSets.main.output + configurations = [project.configurations.runtimeClasspath] +} \ No newline at end of file diff --git a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java index 41f41802e0a..552c646cbe0 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java @@ -260,7 +260,7 @@ public long count(String indexName, QueryBuilder query) { throw new RuntimeException( "Fetching job metadata information failed with status : " + searchResponse.status()); } else { - return searchResponse.getHits().getTotalHits().value(); + return searchResponse.getHits().getTotalHits().value; } } diff --git a/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java index db29921c089..7a655f0678a 100644 --- a/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; -import org.opensearch.action.support.clustermanager.AcknowledgedResponse; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; @RequiredArgsConstructor diff --git a/build.gradle b/build.gradle index 6b7af6ada43..637a025cf03 100644 --- a/build.gradle +++ b/build.gradle @@ -122,12 +122,12 @@ allprojects { resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-annotations:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" resolutionStrategy.force 'com.google.protobuf:protobuf-java:3.25.5' resolutionStrategy.force 'org.locationtech.jts:jts-core:1.19.0' resolutionStrategy.force 'com.google.errorprone:error_prone_annotations:2.28.0' diff --git a/doctest/build.gradle b/doctest/build.gradle index 96bb69231d7..91d54c9cb24 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -100,8 +100,8 @@ task stopPrometheus(type: KillProcessTask) { if(getOSFamilyType() != "windows") { stopPrometheus.mustRunAfter startPrometheus startOpenSearch.dependsOn startPrometheus -// stopOpenSearch.finalizedBy stopPrometheus -// startOpenSearch.finalizedBy stopPrometheus + stopOpenSearch.finalizedBy stopPrometheus + startOpenSearch.finalizedBy stopPrometheus } doctest.dependsOn startOpenSearch doctest.finalizedBy stopOpenSearch diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 79fb637e631..56d54ccb6f6 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -160,12 +160,12 @@ configurations.all { resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-annotations:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.18.2" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.18.2" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.12.0" resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10" @@ -199,7 +199,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Needed for BWC tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java b/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java index 19b5a605ca7..e951c849617 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/domain/SearchResult.java @@ -12,7 +12,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import org.apache.lucene.search.TotalHits; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.document.DocumentField; import org.opensearch.search.SearchHit; @@ -38,7 +37,8 @@ public class SearchResult { public SearchResult(SearchResponse resp) { SearchHits hits = resp.getHits(); - this.total = Optional.ofNullable(hits.getTotalHits()).map(TotalHits::value).orElse(0L); + this.total = + Optional.ofNullable(hits.getTotalHits()).map(totalHits -> totalHits.value).orElse(0L); results = new ArrayList<>(hits.getHits().length); for (SearchHit searchHit : hits.getHits()) { if (searchHit.getSourceAsMap() != null) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index d533e003119..33448298596 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -136,7 +136,7 @@ private Protocol buildProtocolForDefaultQuery(Client client, DefaultQueryAction protected boolean isDefaultCursor(SearchResponse searchResponse, DefaultQueryAction queryAction) { if (LocalClusterState.state().getSettingValue(SQL_PAGINATION_API_SEARCH_AFTER)) { return queryAction.getSqlRequest().fetchSize() != 0 - && Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value() + && Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value >= queryAction.getSqlRequest().fetchSize(); } else { return !Strings.isNullOrEmpty(searchResponse.getScrollId()); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java index e0f6c189069..84b7c00857f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/SelectResultSet.java @@ -26,7 +26,6 @@ import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.search.TotalHits; import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse; import org.opensearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; @@ -542,7 +541,7 @@ private void extractData() { this.rows = populateRows(searchHits); this.size = rows.size(); this.internalTotalHits = - Optional.ofNullable(searchHits.getTotalHits()).map(TotalHits::value).orElse(0L); + Optional.ofNullable(searchHits.getTotalHits()).map(th -> th.value).orElse(0L); // size may be greater than totalHits after nested rows be flatten this.totalHits = Math.max(size, internalTotalHits); } else if (queryResult instanceof Aggregations) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java index 028861aa47c..70e7118ad5f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java @@ -13,7 +13,6 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; -import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; @@ -63,10 +62,7 @@ public static String hitsAsStringResult(SearchHits results, MetaSearchResult met Object[] searchHits; searchHits = new Object - [Optional.ofNullable(results.getTotalHits()) - .map(TotalHits::value) - .orElse(0L) - .intValue()]; + [Optional.ofNullable(results.getTotalHits()).map(th -> th.value).orElse(0L).intValue()]; int i = 0; for (SearchHit hit : results) { HashMap value = new HashMap<>(); @@ -80,10 +76,10 @@ public static String hitsAsStringResult(SearchHits results, MetaSearchResult met hits.put( "total", ImmutableMap.of( - "value", Optional.ofNullable(results.getTotalHits()).map(TotalHits::value).orElse(0L), + "value", Optional.ofNullable(results.getTotalHits()).map(th -> th.value).orElse(0L), "relation", Optional.ofNullable(results.getTotalHits()) - .map(TotalHits::relation) + .map(th -> th.relation) .orElse(Relation.EQUAL_TO))); hits.put("max_score", results.getMaxScore()); hits.put("hits", searchHits); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index 36335db9969..f4e4347e067 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -305,7 +305,7 @@ private FetchWithScrollResponse firstFetch(TableInJoinRequestBuilder tableReques // scroll request with max. responseWithHits = getResponseWithHits(tableRequest, MAX_RESULTS_ON_ONE_FETCH, null); if (responseWithHits.getHits().getTotalHits() != null - && responseWithHits.getHits().getTotalHits().value() < MAX_RESULTS_ON_ONE_FETCH) { + && responseWithHits.getHits().getTotalHits().value < MAX_RESULTS_ON_ONE_FETCH) { needScrollForFirstTable = true; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java index 84530925bc6..20fb52c1125 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java @@ -124,7 +124,7 @@ public boolean pushDownFilter(Filter filter) { QueryBuilder filterBuilder = PredicateAnalyzer.analyze(filter.getCondition(), schema); requestBuilder.pushDownFilter(filterBuilder); // TODO: handle the case where condition contains a score function - return false; + return true; } catch (ExpressionNotAnalyzableException | PredicateAnalyzerException e) { LOG.warn("Cannot analyze the filter condition {}", filter.getCondition(), e); } diff --git a/plugin/build.gradle b/plugin/build.gradle index b0b7a5a8053..9df3d3dd483 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -170,7 +170,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${versions.mockito}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" } test { From 2469c26e98b3e2f736b6855a9fed835a6afc162c Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Tue, 25 Feb 2025 17:17:17 +0800 Subject: [PATCH 3/5] Fix the timestamp issue Signed-off-by: Lantao Jin --- core/build.gradle | 5 +- .../sql/calcite/CalcitePlanContext.java | 16 +- .../sql/calcite/CalciteRelNodeVisitor.java | 6 - .../sql/calcite/utils/CalciteToolsHelper.java | 185 ++++++++++++++++++ .../calcite/utils/OpenSearchTypeFactory.java | 32 +-- .../sql/data/model/AbstractExprValue.java | 5 + .../sql/data/model/ExprDateValue.java | 6 + .../sql/data/model/ExprTimeValue.java | 5 + .../sql/data/model/ExprTimestampValue.java | 5 + .../opensearch/sql/data/model/ExprValue.java | 3 + .../opensearch/sql/executor/QueryService.java | 5 +- .../remote/CalciteFieldsCommandIT.java | 7 - .../remote/CalciteQueryAnalysisIT.java | 47 +++-- .../remote/CalciteSearchCommandIT.java | 24 +-- .../calcite/remote/CalciteStatsCommandIT.java | 17 +- .../calcite/remote/CalciteWhereCommandIT.java | 15 +- .../calcite/standalone/CalcitePPLBasicIT.java | 40 +++- .../executor/OpenSearchExecutionEngine.java | 22 ++- .../opensearch/storage/OpenSearchIndex.java | 3 +- .../scan/OpenSearchIndexEnumerator.java | 4 +- .../util/JdbcOpenSearchDataTypeConvertor.java | 16 +- 21 files changed, 343 insertions(+), 125 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java diff --git a/core/build.gradle b/core/build.gradle index 844941d0683..bf8060abaa9 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -119,15 +119,14 @@ jacocoTestCoverageVerification { 'org.opensearch.sql.datasource.model.DataSource', 'org.opensearch.sql.datasource.model.DataSourceStatus', 'org.opensearch.sql.datasource.model.DataSourceType', - 'org.opensearch.sql.executor.ExecutionEngine' ] limit { counter = 'LINE' - minimum = 0.5 // calcite dev only + minimum = 0.0 // calcite dev only } limit { counter = 'BRANCH' - minimum = 0.5 // calcite dev only + minimum = 0.0 // calcite dev only } } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index e89d006eaa3..775a0b8e9fb 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -5,24 +5,29 @@ package org.opensearch.sql.calcite; +import java.sql.Connection; import java.util.function.BiFunction; import lombok.Getter; +import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; public class CalcitePlanContext { public FrameworkConfig config; + public final Connection connection; public final RelBuilder relBuilder; public final ExtendedRexBuilder rexBuilder; @Getter private boolean isResolvingJoinCondition = false; - public CalcitePlanContext(FrameworkConfig config) { + private CalcitePlanContext(FrameworkConfig config, JavaTypeFactory typeFactory) { this.config = config; - this.relBuilder = RelBuilder.create(config); + this.connection = CalciteToolsHelper.connect(config, typeFactory); + this.relBuilder = CalciteToolsHelper.create(config, typeFactory, connection); this.rexBuilder = new ExtendedRexBuilder(relBuilder.getRexBuilder()); } @@ -35,8 +40,11 @@ public RexNode resolveJoinCondition( return result; } - // for testing only public static CalcitePlanContext create(FrameworkConfig config) { - return new CalcitePlanContext(config); + return new CalcitePlanContext(config, null); + } + + public static CalcitePlanContext create(FrameworkConfig config, JavaTypeFactory typeFactory) { + return new CalcitePlanContext(config, typeFactory); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 75fd8fa5b32..7a5646996ea 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -94,12 +94,6 @@ public RelNode visitProject(Project node, CalcitePlanContext context) { visitChildren(node, context); List projectList; if (node.getProjectList().stream().anyMatch(e -> e instanceof AllFields)) { - // if (context.relBuilder.peek() instanceof TableScan) { - // // force add project all to stack to avoid the TableScan in plan only. - // // if the stack only contains OpenSearchTableScan.ENUMERABLE, it will fail - // // e.g. search source=table should return project(all_fields, child=tableScan) - // context.relBuilder.project(context.relBuilder.fields(), ImmutableList.of(), true); - // } return context.relBuilder.peek(); } else { projectList = diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java new file mode 100644 index 00000000000..a7c51357a60 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -0,0 +1,185 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Properties; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.avatica.AvaticaConnection; +import org.apache.calcite.avatica.AvaticaFactory; +import org.apache.calcite.avatica.UnregisteredDriver; +import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.interpreter.Bindables; +import org.apache.calcite.jdbc.CalciteFactory; +import org.apache.calcite.jdbc.CalciteJdbc41Factory; +import org.apache.calcite.jdbc.CalcitePrepare; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.jdbc.Driver; +import org.apache.calcite.plan.Context; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptSchema; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.prepare.CalciteCatalogReader; +import org.apache.calcite.prepare.CalcitePrepareImpl; +import org.apache.calcite.rel.RelHomogeneousShuttle; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalTableScan; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.server.CalciteServerStatement; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelRunner; +import org.apache.calcite.util.Util; +import org.opensearch.sql.calcite.CalcitePlanContext; + +/** + * Calcite Tools Helper. This class is used to create customized: 1. Connection 2. JavaTypeFactory + * 3. RelBuilder 4. RelRunner TODO delete it in future if possible. + */ +public class CalciteToolsHelper { + + /** Create a RelBuilder with testing */ + public static RelBuilder create(FrameworkConfig config) { + return RelBuilder.create(config); + } + + /** Create a RelBuilder with typeFactory */ + public static RelBuilder create( + FrameworkConfig config, JavaTypeFactory typeFactory, Connection connection) { + return withPrepare( + config, + typeFactory, + connection, + (cluster, relOptSchema, rootSchema, statement) -> + new OpenSearchRelBuilder(config.getContext(), cluster, relOptSchema)); + } + + public static Connection connect(FrameworkConfig config, JavaTypeFactory typeFactory) { + final Properties info = new Properties(); + if (config.getTypeSystem() != RelDataTypeSystem.DEFAULT) { + info.setProperty( + CalciteConnectionProperty.TYPE_SYSTEM.camelName(), + config.getTypeSystem().getClass().getName()); + } + try { + return new OpenSearchDriver().connect("jdbc:calcite:", info, null, typeFactory); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * This method copied from {@link Frameworks#withPrepare(FrameworkConfig, + * Frameworks.BasePrepareAction)}. The purpose is the method {@link + * CalciteFactory#newConnection(UnregisteredDriver, AvaticaFactory, String, Properties)} create + * connection with null instance of JavaTypeFactory. So we add a parameter JavaTypeFactory. + */ + private static R withPrepare( + FrameworkConfig config, + JavaTypeFactory typeFactory, + Connection connection, + Frameworks.BasePrepareAction action) { + try { + final Properties info = new Properties(); + if (config.getTypeSystem() != RelDataTypeSystem.DEFAULT) { + info.setProperty( + CalciteConnectionProperty.TYPE_SYSTEM.camelName(), + config.getTypeSystem().getClass().getName()); + } + final CalciteServerStatement statement = + connection.createStatement().unwrap(CalciteServerStatement.class); + return new OpenSearchPrepareImpl().perform(statement, config, typeFactory, action); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static class OpenSearchDriver extends Driver { + + public Connection connect( + String url, Properties info, CalciteSchema rootSchema, JavaTypeFactory typeFactory) + throws SQLException { + CalciteJdbc41Factory factory = new CalciteJdbc41Factory(); + AvaticaConnection connection = + factory.newConnection((Driver) this, factory, url, info, rootSchema, typeFactory); + this.handler.onConnectionInit(connection); + return connection; + } + } + + /** do nothing, just extend for a public construct for new */ + public static class OpenSearchRelBuilder extends RelBuilder { + public OpenSearchRelBuilder(Context context, RelOptCluster cluster, RelOptSchema relOptSchema) { + super(context, cluster, relOptSchema); + } + } + + public static class OpenSearchPrepareImpl extends CalcitePrepareImpl { + /** + * Similar to {@link CalcitePrepareImpl#perform(CalciteServerStatement, FrameworkConfig, + * Frameworks.BasePrepareAction)}, but with a custom typeFactory. + */ + public R perform( + CalciteServerStatement statement, + FrameworkConfig config, + JavaTypeFactory typeFactory, + Frameworks.BasePrepareAction action) { + final CalcitePrepare.Context prepareContext = statement.createPrepareContext(); + SchemaPlus defaultSchema = config.getDefaultSchema(); + final CalciteSchema schema = + defaultSchema != null + ? CalciteSchema.from(defaultSchema) + : prepareContext.getRootSchema(); + CalciteCatalogReader catalogReader = + new CalciteCatalogReader( + schema.root(), schema.path(null), typeFactory, prepareContext.config()); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final RelOptPlanner planner = + createPlanner(prepareContext, config.getContext(), config.getCostFactory()); + final RelOptCluster cluster = createCluster(planner, rexBuilder); + return action.apply(cluster, catalogReader, prepareContext.getRootSchema().plus(), statement); + } + } + + public static class OpenSearchRelRunners { + /** + * Runs a relational expression by existing connection. This class copied from {@link + * org.apache.calcite.tools.RelRunners#run(RelNode)} + */ + public static PreparedStatement run(CalcitePlanContext context, RelNode rel) { + final RelShuttle shuttle = + new RelHomogeneousShuttle() { + @Override + public RelNode visit(TableScan scan) { + final RelOptTable table = scan.getTable(); + if (scan instanceof LogicalTableScan + && Bindables.BindableTableScan.canHandle(table)) { + // Always replace the LogicalTableScan with BindableTableScan + // because it's implementation does not require a "schema" as context. + return Bindables.BindableTableScan.create(scan.getCluster(), table); + } + return super.visit(scan); + } + }; + rel = rel.accept(shuttle); + // the line we changed here + try (Connection connection = context.connection) { + final RelRunner runner = connection.unwrap(RelRunner.class); + return runner.prepareStatement(rel); + } catch (SQLException e) { + throw Util.throwAsRuntime(e); + } + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 0dfa7d65f78..5beebad2dc3 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.utils; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -113,29 +114,10 @@ public static RelDataType convertSchema(Table table) { } return TYPE_FACTORY.createStructType(typeList, fieldNameList, true); } - // - // @Override - // public Type getJavaClass(RelDataType type) { - // if (type instanceof JavaType) { - // JavaType javaType = (JavaType) type; - // return javaType.getJavaClass(); - // } - // if (type instanceof BasicSqlType || type instanceof IntervalSqlType) { - // switch (type.getSqlTypeName()) { - // case DATE: - // return Date.class; - // case TIME: - // case TIME_WITH_LOCAL_TIME_ZONE: - // case TIME_TZ: - // return Time.class; - // case TIMESTAMP: - // case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - // case TIMESTAMP_TZ: - // return Timestamp.class; - // default: - // break; - // } - // } - // return super.getJavaClass(type); - // } + + /** not in use for now, but let's keep this code for future reference. */ + @Override + public Type getJavaClass(RelDataType type) { + return super.getJavaClass(type); + } } diff --git a/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java index f332867645c..5df9ee6b070 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java @@ -82,4 +82,9 @@ public boolean equals(Object o) { /** The expression value equal. */ public abstract boolean equal(ExprValue other); + + @Override + public Object valueForCalcite() { + return value(); + } } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index c36cd3ea6d0..44418d426ff 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -11,6 +11,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; +import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -41,6 +42,11 @@ public String value() { return DateTimeFormatter.ISO_LOCAL_DATE.format(date); } + @Override + public Long valueForCalcite() { + return date.atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli(); + } + @Override public ExprType type() { return ExprCoreType.DATE; diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index 6b5a4a7c481..ebd61d7736d 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -42,6 +42,11 @@ public String value() { return ISO_LOCAL_TIME.format(time); } + @Override + public Long valueForCalcite() { + return time.toNanoOfDay() / 1000; + } + @Override public ExprType type() { return ExprCoreType.TIME; diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java index e103dc72539..9a4577d7ec3 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java @@ -56,6 +56,11 @@ public String value() { : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); } + @Override + public Long valueForCalcite() { + return timestamp.toEpochMilli(); + } + @Override public ExprType type() { return ExprCoreType.TIMESTAMP; diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java index da9c329f937..5b5f20bf389 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java @@ -23,6 +23,9 @@ public interface ExprValue extends Serializable, Comparable { /** Get the Object value of the Expression Value. */ Object value(); + /** Get the Object for Calcite engine compatibility */ + Object valueForCalcite(); + /** Get the {@link ExprCoreType} of the Expression Value. */ ExprType type(); diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index d9de841e401..7a04ad7deff 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -8,6 +8,8 @@ package org.opensearch.sql.executor; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; + import java.security.AccessController; import java.security.PrivilegedAction; import java.util.List; @@ -78,7 +80,8 @@ public void execute( (PrivilegedAction) () -> { final FrameworkConfig config = buildFrameworkConfig(); - final CalcitePlanContext context = new CalcitePlanContext(config); + final CalcitePlanContext context = + CalcitePlanContext.create(config, TYPE_FACTORY); executePlanByCalcite(analyze(plan, context), context, listener); return null; }); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java index 8884fd45a41..2b07b712468 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldsCommandIT.java @@ -32,11 +32,4 @@ public void testDelimitedMetadataFields() throws IOException { public void testMetadataFields() throws IOException { super.testMetadataFields(); } - - @Override - @Test - @Ignore("https://github.com/opensearch-project/sql/issues/3341") - public void testSelectDateTypeField() throws IOException { - super.testSelectDateTypeField(); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java index 1fb0acfeda1..b04d1e94819 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteQueryAnalysisIT.java @@ -5,35 +5,34 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; + +import java.io.IOException; import org.junit.jupiter.api.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.QueryAnalysisIT; -import java.io.IOException; - -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; - public class CalciteQueryAnalysisIT extends QueryAnalysisIT { - @Override - public void init() throws IOException { - enableCalcite(); - disallowCalciteFallback(); - super.init(); - } + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } - @Override - @Test - public void nonexistentFieldShouldFailSemanticCheck() { - String query = String.format("search source=%s | fields name", TEST_INDEX_ACCOUNT); - try { - executeQuery(query); - fail("Expected to throw Exception, but none was thrown for query: " + query); - } catch (ResponseException e) { - String errorMsg = e.getMessage(); - assertTrue(errorMsg.contains("IllegalArgumentException")); - assertTrue(errorMsg.contains("field [name] not found")); - } catch (IOException e) { - throw new IllegalStateException("Unexpected exception raised for query: " + query); - } + @Override + @Test + public void nonexistentFieldShouldFailSemanticCheck() { + String query = String.format("search source=%s | fields name", TEST_INDEX_ACCOUNT); + try { + executeQuery(query); + fail("Expected to throw Exception, but none was thrown for query: " + query); + } catch (ResponseException e) { + String errorMsg = e.getMessage(); + assertTrue(errorMsg.contains("IllegalArgumentException")); + assertTrue(errorMsg.contains("field [name] not found")); + } catch (IOException e) { + throw new IllegalStateException("Unexpected exception raised for query: " + query); } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java index 4751c98a3fa..19fa22e19d2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java @@ -5,24 +5,14 @@ package org.opensearch.sql.calcite.remote; -import org.junit.Ignore; -import org.junit.jupiter.api.Test; -import org.opensearch.sql.ppl.SearchCommandIT; - import java.io.IOException; +import org.opensearch.sql.ppl.SearchCommandIT; public class CalciteSearchCommandIT extends SearchCommandIT { - @Override - public void init() throws IOException { - enableCalcite(); - disallowCalciteFallback(); - super.init(); - } - - @Override - @Test - @Ignore("https://github.com/opensearch-project/sql/issues/3341") - public void testSearchCommandWithoutSearchKeyword() throws IOException { - super.testSearchCommandWithoutSearchKeyword(); - } + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java index 0a03c07db3a..32723154ae6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java @@ -5,18 +5,17 @@ package org.opensearch.sql.calcite.remote; +import java.io.IOException; import org.junit.Ignore; import org.opensearch.sql.ppl.StatsCommandIT; -import java.io.IOException; - -//TODO +// TODO @Ignore("Not all agg functions are supported in Calcite now") public class CalciteStatsCommandIT extends StatsCommandIT { - @Override - public void init() throws IOException { - enableCalcite(); - disallowCalciteFallback(); - super.init(); - } + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 2905a17a4fa..21ff3d86ea0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -5,17 +5,16 @@ package org.opensearch.sql.calcite.remote; +import java.io.IOException; import org.junit.Ignore; import org.opensearch.sql.ppl.WhereCommandIT; -import java.io.IOException; - @Ignore("Not all boolean functions are supported in Calcite now") public class CalciteWhereCommandIT extends WhereCommandIT { - @Override - public void init() throws IOException { - enableCalcite(); - disallowCalciteFallback(); - super.init(); - } + @Override + public void init() throws IOException { + enableCalcite(); + disallowCalciteFallback(); + super.init(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java index 2a1569cf035..86294034850 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java @@ -294,7 +294,8 @@ public void testFilterQueryWithOr2() { @Test public void testQueryMinusFields() { String actual = - execute(String.format("source=%s | fields - firstname, lastname, birthdate", TEST_INDEX_BANK)); + execute( + String.format("source=%s | fields - firstname, lastname, birthdate", TEST_INDEX_BANK)); assertEquals( "{\n" + " \"schema\": [\n" @@ -594,9 +595,44 @@ public void testMultipleTablesAndFilters() { actual); } - @Ignore // https://github.com/opensearch-project/sql/issues/3341 + @Test public void testSelectDateTypeField() { String actual = execute(String.format("source=%s | fields birthdate", TEST_INDEX_BANK)); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"birthdate\",\n" + + " \"type\": \"timestamp\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"2017-10-23 00:00:00\"\n" + + " ],\n" + + " [\n" + + " \"2017-11-20 00:00:00\"\n" + + " ],\n" + + " [\n" + + " \"2018-06-23 00:00:00\"\n" + + " ],\n" + + " [\n" + + " \"2018-11-13 23:33:20\"\n" + + " ],\n" + + " [\n" + + " \"2018-06-27 00:00:00\"\n" + + " ],\n" + + " [\n" + + " \"2018-08-19 00:00:00\"\n" + + " ],\n" + + " [\n" + + " \"2018-08-11 00:00:00\"\n" + + " ]\n" + + " ],\n" + + " \"total\": 7,\n" + + " \"size\": 7\n" + + "}", + actual); } @Test diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 26debff95dd..f3b47b31198 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -17,10 +17,10 @@ import java.util.Map; import lombok.RequiredArgsConstructor; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.tools.RelRunners; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -39,7 +39,6 @@ /** OpenSearch execution engine implementation. */ @RequiredArgsConstructor public class OpenSearchExecutionEngine implements ExecutionEngine { - private static final Logger LOG = LogManager.getLogger(); private final OpenSearchClient client; @@ -113,9 +112,9 @@ public void execute( AccessController.doPrivileged( (PrivilegedAction) () -> { - try (PreparedStatement statement = RelRunners.run(rel)) { + try (PreparedStatement statement = OpenSearchRelRunners.run(context, rel)) { ResultSet result = statement.executeQuery(); - buildResultSet(result, listener); + buildResultSet(result, rel.getRowType(), listener); return null; } catch (SQLException e) { throw new RuntimeException(e); @@ -123,12 +122,14 @@ public void execute( }); } - private void buildResultSet(ResultSet resultSet, ResponseListener listener) + private void buildResultSet( + ResultSet resultSet, RelDataType rowTypes, ResponseListener listener) throws SQLException { // Get the ResultSet metadata to know about columns ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = metaData.getColumnCount(); - + List fieldTypes = + rowTypes.getFieldList().stream().map(RelDataTypeField::getType).toList(); List values = new ArrayList<>(); // Iterate through the ResultSet while (resultSet.next()) { @@ -137,8 +138,10 @@ private void buildResultSet(ResultSet resultSet, ResponseListener for (int i = 1; i <= columnCount; i++) { String columnName = metaData.getColumnName(i); int sqlType = metaData.getColumnType(i); + RelDataType fieldType = fieldTypes.get(i - 1); ExprValue exprValue = - JdbcOpenSearchDataTypeConvertor.getExprValueFromSqlType(resultSet, i, sqlType); + JdbcOpenSearchDataTypeConvertor.getExprValueFromSqlType( + resultSet, i, sqlType, fieldType); row.put(columnName, exprValue); } values.add(ExprTupleValue.fromExprValueMap(row)); @@ -148,6 +151,7 @@ private void buildResultSet(ResultSet resultSet, ResponseListener for (int i = 1; i <= columnCount; ++i) { String columnName = metaData.getColumnName(i); int sqlType = metaData.getColumnType(i); + RelDataType fieldType = fieldTypes.get(i - 1); ExprType exprType = JdbcOpenSearchDataTypeConvertor.getExprTypeFromSqlType(sqlType); columns.add(new Column(columnName, null, exprType)); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index 19212a662d5..99df0465bdc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -34,7 +34,6 @@ import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalTableScan; -import org.opensearch.sql.opensearch.storage.scan.CalciteOpenSearchIndexScan; import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexEnumerator; import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexScan; import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexScanBuilder; @@ -94,7 +93,7 @@ public OpenSearchIndex(OpenSearchClient client, Settings settings, String indexN @Override public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { final RelOptCluster cluster = context.getCluster(); -// return new CalciteOpenSearchIndexScan(cluster, relOptTable, this); + // return new CalciteOpenSearchIndexScan(cluster, relOptTable, this); return new CalciteLogicalTableScan(cluster, relOptTable, this); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java index c87ea0df5f4..518e67d49fa 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java @@ -54,14 +54,14 @@ private void fetchNextBatch() { OpenSearchResponse response = client.search(request); if (!response.isEmpty()) { iterator = response.iterator(); - } else if (iterator == null){ + } else if (iterator == null) { iterator = Collections.emptyIterator(); } } @Override public Object current() { - Object[] p = fields.stream().map(k -> current.tupleValue().get(k).value()).toArray(); + Object[] p = fields.stream().map(k -> current.tupleValue().get(k).valueForCalcite()).toArray(); return p; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java index d8d3ae9ecac..a0431535eac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java @@ -9,8 +9,12 @@ import java.sql.SQLException; import java.sql.Types; import lombok.experimental.UtilityClass; +import org.apache.calcite.rel.type.RelDataType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprTimeValue; +import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; @@ -49,8 +53,8 @@ public static ExprType getExprTypeFromSqlType(int sqlType) { } } - public static ExprValue getExprValueFromSqlType(ResultSet rs, int i, int sqlType) - throws SQLException { + public static ExprValue getExprValueFromSqlType( + ResultSet rs, int i, int sqlType, RelDataType fieldType) throws SQLException { Object value; switch (sqlType) { case Types.VARCHAR: @@ -75,11 +79,11 @@ public static ExprValue getExprValueFromSqlType(ResultSet rs, int i, int sqlType value = rs.getFloat(i); break; case Types.DATE: - value = rs.getDate(i); - break; + return new ExprDateValue(rs.getString(i)); + case Types.TIME: + return new ExprTimeValue(rs.getString(i)); case Types.TIMESTAMP: - value = rs.getTimestamp(i); - break; + return new ExprTimestampValue(rs.getString(i)); case Types.BOOLEAN: value = rs.getBoolean(i); break; From 4883b3086c20a0325717401dd36b481494d51e4a Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Tue, 25 Feb 2025 19:01:55 +0800 Subject: [PATCH 4/5] address comments Signed-off-by: Lantao Jin --- .../sql/calcite/utils/CalciteToolsHelper.java | 22 ++++ .../calcite/utils/OpenSearchTypeFactory.java | 103 +++++++++++++++++- .../sql/data/model/ExprTupleValue.java | 9 ++ .../sql/calcite/remote/CalciteDataTypeIT.java | 13 +-- .../calcite/standalone/CalcitePPLBasicIT.java | 5 + .../executor/OpenSearchExecutionEngine.java | 5 +- .../physical/EnumerableIndexScanRule.java | 8 +- 7 files changed, 144 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index a7c51357a60..dc62eea9909 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -3,6 +3,28 @@ * SPDX-License-Identifier: Apache-2.0 */ +/* + * This file contains code from the Apache Calcite project (original license below). + * It contains modifications, which are licensed as above: + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.opensearch.sql.calcite.utils; import java.sql.Connection; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 5beebad2dc3..96497d14383 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -13,10 +13,28 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.storage.Table; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.BYTE; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.SHORT; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; + /** This class is used to create RelDataType and map RelDataType to Java data type */ public class OpenSearchTypeFactory extends JavaTypeFactoryImpl { public static final OpenSearchTypeFactory TYPE_FACTORY = @@ -43,12 +61,12 @@ public RelDataType createMapType(RelDataType keyType, RelDataType valueType, boo return createTypeWithNullability(super.createMapType(keyType, valueType), nullable); } - public static RelDataType convertSchemaField(ExprType field) { - return convertSchemaField(field, true); + public static RelDataType convertExprTypeToRelDataType(ExprType field) { + return convertExprTypeToRelDataType(field, true); } /** Converts a OpenSearch ExprCoreType field to relational type. */ - public static RelDataType convertSchemaField(ExprType fieldType, boolean nullable) { + public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boolean nullable) { if (fieldType instanceof ExprCoreType) { switch ((ExprCoreType) fieldType) { case UNDEFINED: @@ -97,7 +115,7 @@ public static RelDataType convertSchemaField(ExprType fieldType, boolean nullabl } else if (fieldType.legacyTypeName().equalsIgnoreCase("text")) { return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable); } else if (fieldType.legacyTypeName().equalsIgnoreCase("ip")) { - return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable); + return TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, nullable); // TODO UDT } else { throw new IllegalArgumentException( "Unsupported conversion for OpenSearch Data type: " + fieldType.typeName()); @@ -105,12 +123,87 @@ public static RelDataType convertSchemaField(ExprType fieldType, boolean nullabl } } + /** Converts a Calcite data type to OpenSearch ExprCoreType. */ + public static ExprType convertRelDataTypeToExprType(RelDataType type) { + switch (type.getSqlTypeName()) { + case TINYINT: + return BYTE; + case SMALLINT: + return SHORT; + case INTEGER: + return INTEGER; + case BIGINT: + return LONG; + case REAL: + return FLOAT; + case DOUBLE: + return DOUBLE; + case VARCHAR: + return STRING; + case BOOLEAN: + return BOOLEAN; + case DATE: + return DATE; + case TIME: + return TIME; + case TIMESTAMP: + return TIMESTAMP; + case GEOMETRY: + return IP; + case ARRAY: + return ARRAY; + case MAP: + return STRUCT; + case NULL: + return UNDEFINED; + default: + throw new IllegalArgumentException( + "Unsupported conversion for Relational Data type: " + type.getSqlTypeName()); + } + } + + public static ExprValue getExprValueByExprType(ExprType type, Object value) { + switch (type) { + case UNDEFINED: + return ExprValueUtils.nullValue(); + case BYTE: + return ExprValueUtils.byteValue((Byte) value); + case SHORT: + return ExprValueUtils.shortValue((Short) value); + case INTEGER: + return ExprValueUtils.integerValue((Integer) value); + case LONG: + return ExprValueUtils.longValue((Long) value); + case FLOAT: + return ExprValueUtils.floatValue((Float) value); + case DOUBLE: + return ExprValueUtils.doubleValue((Double) value); + case STRING: + return ExprValueUtils.stringValue((String) value); + case BOOLEAN: + return ExprValueUtils.booleanValue((Boolean) value); + case DATE: + case TIME: + case TIMESTAMP: + return ExprValueUtils.fromObjectValue(value); + case IP: + return ExprValueUtils.ipValue((String) value); + case ARRAY: + return ExprValueUtils.collectionValue((List) value); + case STRUCT: + return ExprValueUtils.tupleValue((Map) value); + default: + throw new IllegalArgumentException( + "Unsupported conversion for OpenSearch Data type: " + type.typeName()); + } + } + public static RelDataType convertSchema(Table table) { List fieldNameList = new ArrayList<>(); List typeList = new ArrayList<>(); for (Map.Entry entry : table.getFieldTypes().entrySet()) { fieldNameList.add(entry.getKey()); - typeList.add(OpenSearchTypeFactory.convertSchemaField(entry.getValue())); + typeList.add(OpenSearchTypeFactory.convertExprTypeToRelDataType(entry.getValue())); } return TYPE_FACTORY.createStructType(typeList, fieldNameList, true); } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java index 856075bed82..81945fb66d3 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java @@ -37,6 +37,15 @@ public Object value() { return resultMap; } + @Override + public Object valueForCalcite() { + LinkedHashMap resultMap = new LinkedHashMap<>(); + for (Entry entry : valueMap.entrySet()) { + resultMap.put(entry.getKey(), entry.getValue().valueForCalcite()); + } + return resultMap; + } + @Override public ExprType type() { return ExprCoreType.STRUCT; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java index ae03c68bc31..6bda45b1963 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java @@ -6,10 +6,16 @@ package org.opensearch.sql.calcite.remote; import java.io.IOException; + +import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ppl.DataTypeIT; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + public class CalciteDataTypeIT extends DataTypeIT { @Override @@ -25,11 +31,4 @@ public void init() throws IOException { public void test_nonnumeric_data_types() throws IOException { super.test_nonnumeric_data_types(); } - - @Override - @Test - @Ignore("ignore this class since the data type is unmatched in calcite engine") - public void test_numeric_data_types() throws IOException { - super.test_numeric_data_types(); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java index 86294034850..4664391b613 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java @@ -6,8 +6,13 @@ package org.opensearch.sql.calcite.standalone; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; + +import org.json.JSONObject; import org.junit.Ignore; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index f3b47b31198..bd8e78aa4b4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -36,6 +36,8 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.TableScanOperator; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.convertRelDataTypeToExprType; + /** OpenSearch execution engine implementation. */ @RequiredArgsConstructor public class OpenSearchExecutionEngine implements ExecutionEngine { @@ -150,9 +152,8 @@ private void buildResultSet( List columns = new ArrayList<>(metaData.getColumnCount()); for (int i = 1; i <= columnCount; ++i) { String columnName = metaData.getColumnName(i); - int sqlType = metaData.getColumnType(i); RelDataType fieldType = fieldTypes.get(i - 1); - ExprType exprType = JdbcOpenSearchDataTypeConvertor.getExprTypeFromSqlType(sqlType); + ExprType exprType = convertRelDataTypeToExprType(fieldType); columns.add(new Column(columnName, null, exprType)); } Schema schema = new Schema(columns); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java index 69c3653d3bb..69166ff98da 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java @@ -6,22 +6,16 @@ package org.opensearch.sql.opensearch.planner.physical; import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.adapter.enumerable.EnumerableProject; import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.logical.LogicalProject; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalTableScan; import org.opensearch.sql.opensearch.storage.scan.CalciteOpenSearchIndexScan; /** - * Rule to convert a {@link LogicalProject} to an {@link EnumerableProject}. You may provide a - * custom config to convert other nodes that extend {@link Project}. - * - * @see EnumerableRules#ENUMERABLE_PROJECT_RULE + * Rule to convert a {@link CalciteLogicalTableScan} to a {@link CalciteOpenSearchIndexScan}. */ public class EnumerableIndexScanRule extends ConverterRule { /** Default configuration. */ From c3c2f3b4f86ca1ca168ed335ee9e2b2d5368e8eb Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Wed, 26 Feb 2025 10:06:12 +0800 Subject: [PATCH 5/5] fix java style and rename CalcitePPLTestCase back to CalcitePPLIntegTestCase Signed-off-by: Lantao Jin --- .../calcite/utils/OpenSearchTypeFactory.java | 28 +++++++++---------- .../sql/calcite/remote/CalciteDataTypeIT.java | 6 ---- .../standalone/CalcitePPLAggregationIT.java | 2 +- .../calcite/standalone/CalcitePPLBasicIT.java | 7 +---- ...Case.java => CalcitePPLIntegTestCase.java} | 9 +++--- .../calcite/standalone/CalcitePPLSortIT.java | 2 +- .../executor/OpenSearchExecutionEngine.java | 4 +-- .../physical/EnumerableIndexScanRule.java | 5 +--- 8 files changed, 25 insertions(+), 38 deletions(-) rename integ-test/src/test/java/org/opensearch/sql/calcite/standalone/{CalcitePPLTestCase.java => CalcitePPLIntegTestCase.java} (96%) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 96497d14383..eec5447cba1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -5,20 +5,6 @@ package org.opensearch.sql.calcite.utils; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.storage.Table; - import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; @@ -35,6 +21,20 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.storage.Table; + /** This class is used to create RelDataType and map RelDataType to Java data type */ public class OpenSearchTypeFactory extends JavaTypeFactoryImpl { public static final OpenSearchTypeFactory TYPE_FACTORY = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java index 6bda45b1963..934d72ad341 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDataTypeIT.java @@ -6,16 +6,10 @@ package org.opensearch.sql.calcite.remote; import java.io.IOException; - -import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ppl.DataTypeIT; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - public class CalciteDataTypeIT extends DataTypeIT { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java index 85055b06980..ce9dbe8357f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java @@ -12,7 +12,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; -public class CalcitePPLAggregationIT extends CalcitePPLTestCase { +public class CalcitePPLAggregationIT extends CalcitePPLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java index 4664391b613..ca1250b2cf1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java @@ -6,18 +6,13 @@ package org.opensearch.sql.calcite.standalone; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; - -import org.json.JSONObject; import org.junit.Ignore; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; -public class CalcitePPLBasicIT extends CalcitePPLTestCase { +public class CalcitePPLBasicIT extends CalcitePPLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java similarity index 96% rename from integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLTestCase.java rename to integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java index 7a3c78e929b..9a366c2a2a0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java @@ -68,12 +68,13 @@ * This abstract test case provide a standalone env to run PPL query, IT extends this class could * debug the service side execution of PPL in IDE. */ -public abstract class CalcitePPLTestCase extends PPLIntegTestCase { +public abstract class CalcitePPLIntegTestCase extends PPLIntegTestCase { protected PPLService pplService; @Override public void init() throws IOException { - RestHighLevelClient restClient = new CalcitePPLTestCase.InternalRestHighLevelClient(client()); + RestHighLevelClient restClient = + new CalcitePPLIntegTestCase.InternalRestHighLevelClient(client()); OpenSearchClient client = new OpenSearchRestClient(restClient); DataSourceService dataSourceService = new DataSourceServiceImpl( @@ -86,8 +87,8 @@ public void init() throws IOException { ModulesBuilder modules = new ModulesBuilder(); modules.add( - new CalcitePPLTestCase.StandaloneModule( - new CalcitePPLTestCase.InternalRestHighLevelClient(client()), + new CalcitePPLIntegTestCase.StandaloneModule( + new CalcitePPLIntegTestCase.InternalRestHighLevelClient(client()), defaultSettings(), dataSourceService)); Injector injector = modules.createInjector(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java index ad2f9d9686b..6f82afe1680 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLSortIT.java @@ -12,7 +12,7 @@ /** testSortXXAndXX could fail. TODO Remove this @Ignore when the issue fixed. */ // @Ignore -public class CalcitePPLSortIT extends CalcitePPLTestCase { +public class CalcitePPLSortIT extends CalcitePPLIntegTestCase { @Override public void init() throws IOException { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index bd8e78aa4b4..bf34089aaba 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.executor; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.convertRelDataTypeToExprType; + import java.security.AccessController; import java.security.PrivilegedAction; import java.sql.PreparedStatement; @@ -36,8 +38,6 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.TableScanOperator; -import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.convertRelDataTypeToExprType; - /** OpenSearch execution engine implementation. */ @RequiredArgsConstructor public class OpenSearchExecutionEngine implements ExecutionEngine { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java index 69166ff98da..6bddd8f61ce 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableIndexScanRule.java @@ -6,7 +6,6 @@ package org.opensearch.sql.opensearch.planner.physical; import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; @@ -14,9 +13,7 @@ import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalTableScan; import org.opensearch.sql.opensearch.storage.scan.CalciteOpenSearchIndexScan; -/** - * Rule to convert a {@link CalciteLogicalTableScan} to a {@link CalciteOpenSearchIndexScan}. - */ +/** Rule to convert a {@link CalciteLogicalTableScan} to a {@link CalciteOpenSearchIndexScan}. */ public class EnumerableIndexScanRule extends ConverterRule { /** Default configuration. */ public static final Config DEFAULT_CONFIG =