From 562856e796c05a5a10e224fc7581b147cf87afd5 Mon Sep 17 00:00:00 2001 From: Murphy Date: Fri, 21 Jul 2023 13:46:36 +0800 Subject: [PATCH 01/23] tmp test haosu's query Signed-off-by: Murphy --- .../MvRewriteOptimizationTest.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index 7b33d91267f79f..63dbcd8a9ee38e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -2723,6 +2723,99 @@ public void testJoinPredicatePushdown() throws Exception { PlanTestBase.assertContains(plan, "_pushdown_predicate_join_mv1"); } + @Test + public void testJoinPredicatePushdown2() throws Exception { + cluster.runSql("test", "CREATE TABLE `esrc__stays_reservations__335827106c571dc8400c45a1075473f1` (\n" + + " `reservation` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `guest` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `visitor_unique` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `host` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `listing` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `erf_guid` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `date` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `date_checkin_utc` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `date_checkout_utc` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `date_checkin_local` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `date_checkout_local` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + + " `ds` date NULL COMMENT \"Minerva Date Stamp\",\n" + + " `ts` datetime NULL COMMENT \"Minerva Time Stamp\",\n" + + " `dim_guest_count` varchar(65533) NULL COMMENT \"Reservation guest count, accounting for the cumulative changes to the reservation as of the time of the metric. In instances where a reservation is not altered, then this value will be equivalent to the value as of the booking time. For the guest count as of the time of the original booking, use dim_guest_count_at_booking.\",\n" + + " `dim_guest_count_categorical` varchar(65533) NULL COMMENT \"Reservation guest count, segmented, accounting for the cumulative changes to the reservation as of the time of the metric. In instances where a reservation is not altered, then this value will be equivalent to the value as of the booking time. For this dimension as of booking, use the dim_guest_count_categorical_at_booking. If m_guests = 0 (ie. no guests), this dimensions is equal to This value should not exist.\",\n" + + " `dim_trip_nights` varchar(65533) NULL COMMENT \"Number of nights in the reservation. May evolve over time due to alterations. For the number of nights as of the time of the booking, use dim_trip_nights_at_booking.\",\n" + + " `dim_trip_days_in_advance` varchar(65533) NULL COMMENT \"Number of days in advance between reservation event (i.e., booking, alteration, or cancellation) and trip start.\",\n" + + " `dim_trip_nights_long_stays` varchar(65533) NULL COMMENT \"Number of nights in the reservation, segmented for longer stays, accounting for the cumulative changes to the reservation as of the time of the metric. In instances where a reservation is not altered, then this value will be equivalent to the value as of the booking time.\",\n" + + " `net_bookings` double NULL COMMENT \"Minerva Measure\",\n" + + " `net_nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + + " `net_guest_nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + + " `bookings` double NULL COMMENT \"Minerva Measure\",\n" + + " `cancellations` double NULL COMMENT \"Minerva Measure\",\n" + + " `alterations` double NULL COMMENT \"Minerva Measure\",\n" + + " `nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + + " `nights_cancelled` double NULL COMMENT \"Minerva Measure\",\n" + + " `nights_altered` double NULL COMMENT \"Minerva Measure\",\n" + + " `guest_nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + + " `guest_nights_cancelled` double NULL COMMENT \"Minerva Measure\",\n" + + " `guest_nights_altered` double NULL COMMENT \"Minerva Measure\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`reservation`, `guest`, `visitor_unique`, `host`, `listing`, `erf_guid`, `date`, `date_checkin_utc`, `date_checkout_utc`, `date_checkin_local`, `date_checkout_local`)\n" + + "COMMENT \"Minerva Airbnb Measure Source: stays_reservations\"\n" + + "PARTITION BY RANGE(`ds`)\n" + + "(PARTITION p20080807 VALUES [(\"0000-01-01\"), (\"2008-08-07\")),\n" + + "PARTITION p20230620 VALUES [(\"2023-05-21\"), (\"2023-06-20\")),\n" + + "PARTITION p_ds2023062020230719 VALUES [(\"2023-06-20\"), (\"2023-07-20\")))\n" + + "DISTRIBUTED BY HASH(`reservation`) BUCKETS 50\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"colocate_with\" = \"reservation_v5\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"); + cluster.runSql("test", "CREATE TABLE `dsrc__guest_age_groups__2b0ed629a2bbb51869caf8870198d587` (\n" + + " `guest` varchar(65533) NULL COMMENT \"\",\n" + + " `dim_guest_age_group` varchar(65533) NULL COMMENT \"\",\n" + + " `dim_guest_is_senior` varchar(65533) NULL COMMENT \"\",\n" + + " `dim_guest_generation_name` varchar(65533) NULL COMMENT \"\",\n" + + " `dim_guest_is_millennial` varchar(65533) NULL COMMENT \"\",\n" + + " `dim_guest_is_gen_x` varchar(65533) NULL COMMENT \"\",\n" + + " `dim_guest_is_boomer` varchar(65533) NULL COMMENT \"\",\n" + + " `ds` date NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`guest`)\n" + + "DISTRIBUTED BY HASH(guest) \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"); + createAndRefreshMv("test", "_hao_test_mv", "CREATE MATERIALIZED VIEW `_hao_test_mv` \n" + + "COMMENT \"MATERIALIZED_VIEW\"\n" + + "DISTRIBUTED BY HASH(`listing`) BUCKETS 18\n" + + "REFRESH MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"replicated_storage\" = \"true\",\n" + + "\"storage_medium\" = \"HDD\"\n" + + ")\n" + + "AS SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`listing`, `stays_reservations`.`host`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`, `stays_reservations`.`ts`, `stays_reservations`.`ds`, coalesce(`guest_age_groups`.`dim_guest_age_group`, '-unknown-') AS `dim_guest_age_group`, coalesce(`guest_age_groups`.`dim_guest_is_senior`, 'False') AS `dim_guest_is_senior`, coalesce(`guest_age_groups`.`dim_guest_generation_name`, '-unknown-') AS `dim_guest_generation_name`, coalesce(`guest_age_groups`.`dim_guest_is_millennial`, 'False') AS `dim_guest_is_millennial`, coalesce(`guest_age_groups`.`dim_guest_is_gen_x`, 'False') AS `dim_guest_is_gen_x`, coalesce(`guest_age_groups`.`dim_guest_is_boomer`, 'False') AS `dim_guest_is_boomer`, `guest_age_groups`.`ds` AS `guest_age_groups.ds`\n" + + "FROM (SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`host`, `stays_reservations`.`listing`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`ds`, `stays_reservations`.`ts`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`\n" + + "FROM `esrc__stays_reservations__335827106c571dc8400c45a1075473f1` AS `stays_reservations`) stays_reservations LEFT OUTER JOIN (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + + "FROM (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + + "FROM `dsrc__guest_age_groups__2b0ed629a2bbb51869caf8870198d587` AS `guest_age_groups`) guest_age_groups) guest_age_groups ON (((`stays_reservations`.`guest` IS NOT NULL) AND (`stays_reservations`.`guest` != '0')) AND (`stays_reservations`.`guest` = `guest_age_groups`.`guest`)) AND ((substring(CAST(`stays_reservations`.`ts` AS VARCHAR(65533)), 1, 10)) = `guest_age_groups`.`ds`);"); + + String query = + "select * from (SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`listing`, `stays_reservations`.`host`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`, `stays_reservations`.`ts`, `stays_reservations`.`ds`, coalesce(`guest_age_groups`.`dim_guest_age_group`, '-unknown-') AS `dim_guest_age_group`, coalesce(`guest_age_groups`.`dim_guest_is_senior`, 'False') AS `dim_guest_is_senior`, coalesce(`guest_age_groups`.`dim_guest_generation_name`, '-unknown-') AS `dim_guest_generation_name`, coalesce(`guest_age_groups`.`dim_guest_is_millennial`, 'False') AS `dim_guest_is_millennial`, coalesce(`guest_age_groups`.`dim_guest_is_gen_x`, 'False') AS `dim_guest_is_gen_x`, coalesce(`guest_age_groups`.`dim_guest_is_boomer`, 'False') AS `dim_guest_is_boomer`, `guest_age_groups`.`ds` AS `guest_age_groups.ds`\n" + + "FROM (SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`host`, `stays_reservations`.`listing`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`ds`, `stays_reservations`.`ts`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`\n" + + "FROM `esrc__stays_reservations__335827106c571dc8400c45a1075473f1` AS `stays_reservations`) stays_reservations LEFT OUTER JOIN (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + + "FROM (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + + "FROM `dsrc__guest_age_groups__2b0ed629a2bbb51869caf8870198d587` AS `guest_age_groups`) guest_age_groups) guest_age_groups ON (((`stays_reservations`.`guest` IS NOT NULL) AND (`stays_reservations`.`guest` != '0')) AND (`stays_reservations`.`guest` = `guest_age_groups`.`guest`)) AND ((substring(CAST(`stays_reservations`.`ts` AS VARCHAR(65533)), 1, 10)) = `guest_age_groups`.`ds`)) _ limit 100;"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "_hao_test_mv"); + } + @Ignore("outer join and pushdown predicate does not work") @Test public void testJoinPredicatePushdown1() throws Exception { From 412ba698157ca5468d40b9482fb70e98745f59fa Mon Sep 17 00:00:00 2001 From: Murphy <96611012+mofeiatwork@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:17:58 +0800 Subject: [PATCH 02/23] [Feature] support meta functions (#28094) Add function to diagnose materialized view: ```sql -- inspect materialized view metadata select inspect_mv_meta('mv_name'); -- inspect related materialized views of a table select inspect_related_mv('table_name'); -- inspect hive partition info select inspect_hive_part_info('table_name'); ``` (cherry picked from commit 9eea14e87b8aad66867e93a8fc76e8609a4aecdc) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java # fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java # fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java # fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java # fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java # fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java --- .../com/starrocks/analysis/TableName.java | 27 ++ .../java/com/starrocks/catalog/Database.java | 9 + .../starrocks/catalog/MaterializedView.java | 72 ++++ .../starrocks/connector/hive/Partition.java | 12 + .../com/starrocks/server/GlobalStateMgr.java | 15 + .../com/starrocks/server/MetadataMgr.java | 71 ++++ .../sql/analyzer/ExpressionAnalyzer.java | 4 + .../optimizer/rewrite/ConstantFunction.java | 5 + .../rewrite/ScalarOperatorEvaluator.java | 28 +- .../rewrite/ScalarOperatorFunctions.java | 308 +++++++++++++++++- .../sql/plan/ConstantExpressionTest.java | 53 ++- 11 files changed, 599 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java b/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java index b64c9bb12a3a89..da5142f1c65e51 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java @@ -22,6 +22,7 @@ package com.starrocks.analysis; import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.gson.annotations.SerializedName; import com.starrocks.cluster.ClusterNamespace; @@ -35,10 +36,16 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.server.CatalogMgr; import com.starrocks.sql.analyzer.SemanticException; +<<<<<<< HEAD +======= +import com.starrocks.sql.parser.NodePosition; +import org.apache.commons.lang3.StringUtils; +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.List; import java.util.Objects; public class TableName implements Writable, GsonPreProcessable, GsonPostProcessable { @@ -64,6 +71,26 @@ public TableName(String catalog, String db, String tbl) { this.tbl = tbl; } + public static TableName fromString(String name) { + List pieces = Splitter.on(".").splitToList(name); + String catalog = ConnectContext.get().getCurrentCatalog(); + String db = ConnectContext.get().getDatabase(); + if (pieces.isEmpty()) { + throw new IllegalArgumentException("empty table name"); + } else if (pieces.size() == 1) { + if (StringUtils.isEmpty(db)) { + throw new IllegalArgumentException("no database"); + } + return new TableName(catalog, db, pieces.get(0)); + } else if (pieces.size() == 2) { + return new TableName(catalog, pieces.get(0), pieces.get(1)); + } else if (pieces.size() == 3) { + return new TableName(pieces.get(0), pieces.get(1), pieces.get(2)); + } else { + throw new IllegalArgumentException("illegal table name: " + name); + } + } + public void analyze(Analyzer analyzer) throws AnalysisException { if (Strings.isNullOrEmpty(catalog)) { catalog = analyzer.getDefaultCatalog(); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java index b87eece7282fdd..b23b8cae26fde9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java @@ -57,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -580,6 +581,14 @@ public Set getTableNamesViewWithLock() { } } + public Optional tryGetTable(String tableName) { + return Optional.ofNullable(nameToTable.get(tableName)); + } + + public Optional
tryGetTable(long tableId) { + return Optional.ofNullable(idToTable.get(tableId)); + } + public Table getTable(String tableName) { if (nameToTable.containsKey(tableName)) { return nameToTable.get(tableName); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java index ef36c1ed70c63d..6d5189b4341bac 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java @@ -1191,4 +1191,76 @@ public void setPlanContext(MVRewriteContextCache mvRewriteContextCache) { this.mvRewriteContextCache = mvRewriteContextCache; } +<<<<<<< HEAD +======= + /** + * Infer the distribution info based on tables and MV query. + * Currently is max{bucket_num of base_table} + * TODO: infer the bucket number according to MV pattern and cardinality + */ + @Override + public void inferDistribution(DistributionInfo info) throws DdlException { + if (info.getBucketNum() == 0) { + int inferredBucketNum = 0; + for (BaseTableInfo base : getBaseTableInfos()) { + if (base.getTable().isNativeTableOrMaterializedView()) { + OlapTable olapTable = (OlapTable) base.getTable(); + DistributionInfo dist = olapTable.getDefaultDistributionInfo(); + inferredBucketNum = Math.max(inferredBucketNum, dist.getBucketNum()); + } + } + if (inferredBucketNum == 0) { + inferredBucketNum = CatalogUtils.calBucketNumAccordingToBackends(); + } + info.setBucketNum(inferredBucketNum); + } + } + + @Override + public Map getProperties() { + Map properties = super.getProperties(); + // For materialized view, add into session variables into properties. + if (super.getTableProperty() != null && super.getTableProperty().getProperties() != null) { + for (Map.Entry entry : super.getTableProperty().getProperties().entrySet()) { + if (entry.getKey().startsWith(PropertyAnalyzer.PROPERTIES_MATERIALIZED_VIEW_SESSION_PREFIX)) { + String varKey = entry.getKey().substring( + PropertyAnalyzer.PROPERTIES_MATERIALIZED_VIEW_SESSION_PREFIX.length()); + properties.put(varKey, entry.getValue()); + } + } + } + return properties; + } + + @Override + public void gsonPreProcess() throws IOException { + this.serializedPartitionRefTableExprs = new ArrayList<>(); + if (partitionRefTableExprs != null) { + for (Expr partitionExpr : partitionRefTableExprs) { + if (partitionExpr != null) { + serializedPartitionRefTableExprs.add( + new GsonUtils.ExpressionSerializedObject(partitionExpr.toSql())); + } + } + } + } + + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + partitionRefTableExprs = new ArrayList<>(); + if (serializedPartitionRefTableExprs != null) { + for (GsonUtils.ExpressionSerializedObject expressionSql : serializedPartitionRefTableExprs) { + if (expressionSql != null) { + partitionRefTableExprs.add( + SqlParser.parseSqlToExpr(expressionSql.expressionSql, SqlModeHelper.MODE_DEFAULT)); + } + } + } + } + + public String inspectMeta() { + return GsonUtils.GSON.toJson(this); + } +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) } diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/Partition.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/Partition.java index b7893e42c53951..9cfee723cc3b41 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/hive/Partition.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/hive/Partition.java @@ -2,7 +2,9 @@ package com.starrocks.connector.hive; +import com.google.gson.JsonObject; import com.starrocks.connector.PartitionInfo; +import com.starrocks.persist.gson.GsonUtils; import java.util.Map; import java.util.Objects; @@ -94,6 +96,16 @@ public String toString() { return sb.toString(); } + public JsonObject toJson() { + JsonObject obj = new JsonObject(); + obj.add("parameters", (GsonUtils.GSON.toJsonTree(parameters))); + obj.add("inputFormat", (GsonUtils.GSON.toJsonTree(inputFormat))); + obj.add("textFileFormatDesc", (GsonUtils.GSON.toJsonTree(textFileFormatDesc))); + obj.add("fullPath", GsonUtils.GSON.toJsonTree(fullPath)); + obj.add("isSplittable", GsonUtils.GSON.toJsonTree(isSplittable)); + return obj; + } + public static Builder builder() { return new Builder(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index 29c1c14dda16a2..7112dd9da7dbfd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -2646,6 +2646,21 @@ public Database getDb(String name) { return localMetastore.getDb(name); } +<<<<<<< HEAD +======= + public Optional
mayGetTable(long dbId, long tableId) { + return mayGetDb(dbId).flatMap(db -> db.tryGetTable(tableId)); + } + + public Optional mayGetDb(String name) { + return Optional.ofNullable(localMetastore.getDb(name)); + } + + public Optional mayGetDb(long dbId) { + return Optional.ofNullable(localMetastore.getDb(dbId)); + } + +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) public Database getDb(long dbId) { return localMetastore.getDb(dbId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java index 68b1d99b520dbd..c997cb56a9a343 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java @@ -110,6 +110,77 @@ public List listTableNames(String catalogName, String dbName) { return ImmutableList.copyOf(tableNames.build()); } +<<<<<<< HEAD +======= + public boolean createTable(CreateTableStmt stmt) throws DdlException { + String catalogName = stmt.getCatalogName(); + Optional connectorMetadata = getOptionalMetadata(catalogName); + + if (connectorMetadata.isPresent()) { + if (!CatalogMgr.isInternalCatalog(catalogName)) { + String dbName = stmt.getDbName(); + String tableName = stmt.getTableName(); + if (getDb(catalogName, dbName) == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); + } + + if (listTableNames(catalogName, dbName).contains(tableName)) { + if (stmt.isSetIfNotExists()) { + LOG.info("create table[{}] which already exists", tableName); + return false; + } else { + ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); + } + } + } + return connectorMetadata.get().createTable(stmt); + } else { + throw new DdlException("Invalid catalog " + catalogName + " , ConnectorMetadata doesn't exist"); + } + } + + public void dropTable(String catalogName, String dbName, String tblName) { + TableName tableName = new TableName(catalogName, dbName, tblName); + DropTableStmt dropTableStmt = new DropTableStmt(false, tableName, false); + dropTable(dropTableStmt); + } + + public void dropTable(DropTableStmt stmt) { + String catalogName = stmt.getCatalogName(); + String dbName = stmt.getDbName(); + String tableName = stmt.getTableName(); + + Optional connectorMetadata = getOptionalMetadata(catalogName); + connectorMetadata.ifPresent(metadata -> { + try { + metadata.dropTable(stmt); + } catch (DdlException e) { + LOG.error("Failed to drop table {}.{}.{}", catalogName, dbName, tableName, e); + throw new StarRocksConnectorException("Failed to drop table {}.{}.{}", catalogName, dbName, tableName); + } + }); + } + + public Optional
getTable(TableName tableName) { + return Optional.ofNullable(getTable(tableName.getCatalog(), tableName.getDb(), tableName.getTbl())); + } + + public Table getTable(String catalogName, String dbName, String tblName) { + Optional connectorMetadata = getOptionalMetadata(catalogName); + Table connectorTable = connectorMetadata.map(metadata -> metadata.getTable(dbName, tblName)).orElse(null); + if (connectorTable != null) { + // Load meta information from ConnectorTblMetaInfoMgr for each external table. + connectorTblMetaInfoMgr.setTableInfoForConnectorTable(catalogName, dbName, connectorTable); + } + return connectorTable; + } + + public Pair getMaterializedViewIndex(String catalogName, String dbName, String tblName) { + Optional connectorMetadata = getOptionalMetadata(catalogName); + return connectorMetadata.map(metadata -> metadata.getMaterializedViewIndex(dbName, tblName)).orElse(null); + } + +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) public List listPartitionNames(String catalogName, String dbName, String tableName) { Optional connectorMetadata = getOptionalMetadata(catalogName); ImmutableSet.Builder partitionNames = ImmutableSet.builder(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java index 72a5c96c9f396b..bf5400580a3c5c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java @@ -69,6 +69,7 @@ import com.starrocks.sql.common.TypeManager; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; +import com.starrocks.sql.optimizer.rewrite.ScalarOperatorEvaluator; import com.starrocks.sql.optimizer.transformer.ExpressionMapping; import com.starrocks.sql.optimizer.transformer.SqlToScalarOperatorTranslator; @@ -840,6 +841,9 @@ public Void visitFunctionCall(FunctionCallExpr node, Scope scope) { if (fn == null) { fn = AnalyzerUtils.getUdfFunction(session, node.getFnName(), argumentTypes); } + if (fn == null) { + fn = ScalarOperatorEvaluator.INSTANCE.getMetaFunction(node.getFnName(), argumentTypes); + } if (fn == null) { throw new SemanticException("No matching function with signature: %s(%s).", diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ConstantFunction.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ConstantFunction.java index bf6b9acef89fdd..56d388f2fc5759 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ConstantFunction.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ConstantFunction.java @@ -18,6 +18,11 @@ PrimitiveType returnType(); + /** + * These functions are used to inspect metadata of database objects + */ + boolean isMetaFunction() default false; + @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) @interface List { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java index 34de9ee54c2a8b..c0ffe593c3c794 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.starrocks.analysis.FunctionName; import com.starrocks.catalog.Function; import com.starrocks.catalog.FunctionSet; import com.starrocks.catalog.PrimitiveType; @@ -14,6 +15,8 @@ import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.common.ErrorType; +import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.scalar.CallOperator; import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; @@ -79,10 +82,21 @@ private void registerFunction(ImmutableMap.Builder TIME_SLICE_UNIT_MAPPING; + + static { + for (int shiftBy = 0; shiftBy < CONSTANT_128; ++shiftBy) { + INT_128_MASK1_ARR1[shiftBy] = INT_128_OPENER.subtract(BigInteger.ONE).shiftRight(shiftBy + 1); + } + + TIME_SLICE_UNIT_MAPPING = ImmutableMap.builder() + .put("second", ChronoUnit.SECONDS) + .put("minute", ChronoUnit.MINUTES) + .put("hour", ChronoUnit.HOURS) + .put("day", ChronoUnit.DAYS) + .put("month", ChronoUnit.MONTHS) + .put("year", ChronoUnit.YEARS) + .put("week", ChronoUnit.WEEKS) + .put("quarter", IsoFields.QUARTER_YEARS) + .build(); + } + +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) /** * date and time function */ @@ -316,7 +368,16 @@ public static ConstantOperator fromUnixTime(ConstantOperator unixTime, ConstantO return dateFormat(dl, fmtLiteral); } +<<<<<<< HEAD @ConstantFunction(name = "now", argTypes = {}, returnType = DATETIME) +======= + @ConstantFunction.List(list = { + @ConstantFunction(name = "now", argTypes = {}, returnType = DATETIME), + @ConstantFunction(name = "current_timestamp", argTypes = {}, returnType = DATETIME), + @ConstantFunction(name = "localtime", argTypes = {}, returnType = DATETIME), + @ConstantFunction(name = "localtimestamp", argTypes = {}, returnType = DATETIME) + }) +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) public static ConstantOperator now() { ConnectContext connectContext = ConnectContext.get(); LocalDateTime startTime = Instant.ofEpochMilli(connectContext.getStartTime()) @@ -364,6 +425,145 @@ public static ConstantOperator utcTimestamp() { return ConstantOperator.createDatetime(utcStartTime); } +<<<<<<< HEAD +======= + @ConstantFunction(name = "next_day", argTypes = {DATETIME, VARCHAR}, returnType = DATE) + public static ConstantOperator nextDay(ConstantOperator date, ConstantOperator dow) { + int dateDowValue = date.getDate().getDayOfWeek().getValue(); + switch (dow.getVarchar()) { + case "Sunday": + case "Sun": + case "Su": + return ConstantOperator.createDate(date.getDate().plusDays((13L - dateDowValue) % 7 + 1L)); + case "Monday": + case "Mon": + case "Mo": + return ConstantOperator.createDate(date.getDate().plusDays((7L - dateDowValue) % 7 + 1L)); + case "Tuesday": + case "Tue": + case "Tu": + return ConstantOperator.createDate(date.getDate().plusDays((8L - dateDowValue) % 7 + 1L)); + case "Wednesday": + case "Wed": + case "We": + return ConstantOperator.createDate(date.getDate().plusDays((9L - dateDowValue) % 7 + 1L)); + case "Thursday": + case "Thu": + case "Th": + return ConstantOperator.createDate(date.getDate().plusDays((10L - dateDowValue) % 7 + 1L)); + case "Friday": + case "Fri": + case "Fr": + return ConstantOperator.createDate(date.getDate().plusDays((11L - dateDowValue) % 7 + 1L)); + case "Saturday": + case "Sat": + case "Sa": + return ConstantOperator.createDate(date.getDate().plusDays((12L - dateDowValue) % 7 + 1L)); + default: + throw new IllegalArgumentException(dow + " not supported in next_day dow_string"); + } + } + + @ConstantFunction(name = "previous_day", argTypes = {DATETIME, VARCHAR}, returnType = DATE) + public static ConstantOperator previousDay(ConstantOperator date, ConstantOperator dow) { + int dateDowValue = date.getDate().getDayOfWeek().getValue(); + switch (dow.getVarchar()) { + case "Sunday": + case "Sun": + case "Su": + return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue - 1L) % 7 + 1L)); + case "Monday": + case "Mon": + case "Mo": + return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 5L) % 7 + 1L)); + case "Tuesday": + case "Tue": + case "Tu": + return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 4L) % 7 + 1L)); + case "Wednesday": + case "Wed": + case "We": + return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 3L) % 7 + 1L)); + case "Thursday": + case "Thu": + case "Th": + return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 2L) % 7 + 1L)); + case "Friday": + case "Fri": + case "Fr": + return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 1L) % 7 + 1L)); + case "Saturday": + case "Sat": + case "Sa": + return ConstantOperator.createDate(date.getDate().minusDays(dateDowValue % 7 + 1L)); + default: + throw new IllegalArgumentException(dow + " not supported in previous_day dow_string"); + } + } + + @ConstantFunction(name = "makedate", argTypes = {INT, INT}, returnType = DATETIME) + public static ConstantOperator makeDate(ConstantOperator year, ConstantOperator dayOfYear) { + if (year.isNull() || dayOfYear.isNull()) { + return ConstantOperator.createNull(Type.DATE); + } + + int yearInt = year.getInt(); + if (yearInt < YEAR_MIN || yearInt > YEAR_MAX) { + return ConstantOperator.createNull(Type.DATE); + } + + int dayOfYearInt = dayOfYear.getInt(); + if (dayOfYearInt < DAY_OF_YEAR_MIN || dayOfYearInt > DAY_OF_YEAR_MAX) { + return ConstantOperator.createNull(Type.DATE); + } + + LocalDate ld = LocalDate.of(yearInt, 1, 1) + .plusDays(dayOfYearInt - 1); + + if (ld.getYear() != year.getInt()) { + return ConstantOperator.createNull(Type.DATE); + } + + return ConstantOperator.createDate(ld.atTime(0, 0, 0)); + } + + @ConstantFunction(name = "time_slice", argTypes = {DATETIME, INT, VARCHAR}, returnType = DATETIME) + public static ConstantOperator timeSlice(ConstantOperator datetime, ConstantOperator interval, + ConstantOperator unit) throws AnalysisException { + return timeSlice(datetime, interval, unit, ConstantOperator.createVarchar("floor")); + } + + @ConstantFunction(name = "time_slice", argTypes = {DATETIME, INT, VARCHAR, VARCHAR}, returnType = DATETIME) + public static ConstantOperator timeSlice(ConstantOperator datetime, ConstantOperator interval, + ConstantOperator unit, ConstantOperator boundary) + throws AnalysisException { + TemporalUnit timeUnit = TIME_SLICE_UNIT_MAPPING.get(unit.getVarchar()); + if (timeUnit == null) { + throw new IllegalArgumentException(unit + " not supported in time_slice unit param"); + } + boolean isEnd; + switch (boundary.getVarchar()) { + case "floor": + isEnd = false; + break; + case "ceil": + isEnd = true; + break; + default: + throw new IllegalArgumentException(boundary + " not supported in time_slice boundary param"); + } + long duration = TIME_SLICE_START.until(datetime.getDatetime(), timeUnit); + if (duration < 0) { + throw new AnalysisException("time used with time_slice can't before 0001-01-01 00:00:00"); + } + long epoch = duration - (duration % interval.getInt()); + if (isEnd) { + epoch += interval.getInt(); + } + return ConstantOperator.createDatetime(TIME_SLICE_START.plus(epoch, timeUnit)); + } + +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) /** * Math function */ @@ -785,7 +985,8 @@ public static ConstantOperator substring(ConstantOperator value, ConstantOperato /// Besides, the implementation of `substring` function in starrocks includes beginIndex and length, /// and the index is start from 1 and can negative, so we need carefully handle it. int beginIndex = index[0].getInt() >= 0 ? index[0].getInt() - 1 : string.length() + index[0].getInt(); - int endIndex = (index.length == 2) ? Math.min(beginIndex + index[1].getInt(), string.length()) : string.length(); + int endIndex = + (index.length == 2) ? Math.min(beginIndex + index[1].getInt(), string.length()) : string.length(); if (beginIndex < 0 || beginIndex > endIndex) { return ConstantOperator.createVarchar(""); @@ -828,4 +1029,109 @@ private static BigInteger bitShiftRightLogicalForInt128(BigInteger l, int shiftB BigInteger opened = l.subtract(INT_128_OPENER); return opened.shiftRight(shiftBy).and(INT_128_MASK1_ARR1[shiftBy]); } + + // =================================== meta functions ==================================== // + + private static Table inspectExternalTable(TableName tableName) { + Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(tableName) + .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName)); + ConnectContext connectContext = ConnectContext.get(); + PrivilegeChecker.checkAnyActionOnTable( + connectContext.getCurrentUserIdentity(), + connectContext.getCurrentRoleIds(), + tableName); + return table; + } + + private static Pair inspectTable(TableName tableName) { + Database db = GlobalStateMgr.getCurrentState().mayGetDb(tableName.getDb()) + .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_DB_ERROR, tableName.getDb())); + Table table = db.tryGetTable(tableName.getTbl()) + .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName)); + ConnectContext connectContext = ConnectContext.get(); + PrivilegeChecker.checkAnyActionOnTable( + connectContext.getCurrentUserIdentity(), + connectContext.getCurrentRoleIds(), + tableName); + return Pair.of(db, table); + } + + /** + * Return verbose metadata of a materialized-view + */ + @ConstantFunction(name = "inspect_mv_meta", argTypes = {VARCHAR}, returnType = VARCHAR, isMetaFunction = true) + public static ConstantOperator inspect_mv_meta(ConstantOperator mvName) { + TableName tableName = TableName.fromString(mvName.getVarchar()); + Pair dbTable = inspectTable(tableName); + Table table = dbTable.getRight(); + if (!table.isMaterializedView()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_INVALID_PARAMETER, + tableName + " is not materialized view"); + } + try { + dbTable.getLeft().readLock(); + + MaterializedView mv = (MaterializedView) table; + String meta = mv.inspectMeta(); + return ConstantOperator.createVarchar(meta); + } finally { + dbTable.getLeft().readUnlock(); + } + } + + /** + * Return related materialized-views of a table, in JSON array format + */ + @ConstantFunction(name = "inspect_related_mv", argTypes = {VARCHAR}, returnType = VARCHAR, isMetaFunction = true) + public static ConstantOperator inspect_related_mv(ConstantOperator name) { + TableName tableName = TableName.fromString(name.getVarchar()); + Pair dbTable = inspectTable(tableName); + Table table = dbTable.getRight(); + + try { + dbTable.getLeft().readLock(); + + Set relatedMvs = table.getRelatedMaterializedViews(); + JsonArray array = new JsonArray(); + for (MvId mv : SetUtils.emptyIfNull(relatedMvs)) { + String mvName = GlobalStateMgr.getCurrentState().mayGetTable(mv.getDbId(), mv.getId()) + .map(Table::getName) + .orElse(null); + JsonObject obj = new JsonObject(); + obj.add("id", new JsonPrimitive(mv.getId())); + obj.add("name", mvName != null ? new JsonPrimitive(mvName) : JsonNull.INSTANCE); + + array.add(obj); + } + + String json = array.toString(); + return ConstantOperator.createVarchar(json); + } finally { + dbTable.getLeft().readUnlock(); + } + } + + /** + * Return Hive partition info + */ + @ConstantFunction(name = "inspect_hive_part_info", + argTypes = {VARCHAR}, + returnType = VARCHAR, + isMetaFunction = true) + public static ConstantOperator inspect_hive_part_info(ConstantOperator name) { + TableName tableName = TableName.fromString(name.getVarchar()); + Table table = inspectExternalTable(tableName); + + Map info = PartitionUtil.getPartitionNameWithPartitionInfo(table); + JsonObject obj = new JsonObject(); + for (Map.Entry entry : MapUtils.emptyIfNull(info).entrySet()) { + if (entry.getValue() instanceof Partition) { + Partition part = (Partition) entry.getValue(); + obj.add(entry.getKey(), part.toJson()); + } + } + String json = obj.toString(); + return ConstantOperator.createVarchar(json); + } + } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java index 5a215d878962ac..ca0a3ba341f6e6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java @@ -17,19 +17,68 @@ package com.starrocks.sql.plan; +<<<<<<< HEAD +======= +import com.starrocks.qe.SqlModeHelper; +import com.starrocks.sql.common.StarRocksPlannerException; +>>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) import com.starrocks.sql.parser.ParsingException; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class ConstantExpressionTest extends PlanTestBase { + + @BeforeClass + public static void beforeClass() throws Exception { + PlanTestBase.beforeClass(); + ConnectorPlanTestBase.mockHiveCatalog(connectContext); + } + private void testFragmentPlanContainsConstExpr(String sql, String result) throws Exception { String explainString = getFragmentPlan(sql); - Assert.assertTrue(explainString.contains(": " + result)); + Assert.assertTrue(explainString, explainString.contains(": " + result)); } private void testFragmentPlanContains(String sql, String result) throws Exception { String explainString = getFragmentPlan(sql); - Assert.assertTrue(explainString.contains(result)); + Assert.assertTrue(explainString, explainString.contains(result)); + } + + @Test + public void testInspectMvMeta() throws Exception { + String db = starRocksAssert.getCtx().getDatabase(); + starRocksAssert.withTable( + "create table mv_base_table_9527 (id int, name string) properties('replication_num'='1')"); + starRocksAssert.withMaterializedView("create materialized view mv1 " + + "distributed by hash(id) " + + "refresh async " + + "properties('replication_num'='1') " + + "as select * from mv_base_table_9527"); + testFragmentPlanContains("select inspect_mv_meta('mv1');", "MaterializedView"); + String fullName = db + ".mv1"; + testFragmentPlanContains(String.format("select inspect_mv_meta('%s');", fullName), "MaterializedView"); + + // wrong arguments + Assert.assertThrows(StarRocksPlannerException.class, + () -> getFragmentPlan("select inspect_mv_meta('snowflake');")); + Assert.assertThrows(StarRocksPlannerException.class, + () -> getFragmentPlan("select inspect_mv_meta('mv_base_table_9527');")); + Assert.assertThrows(StarRocksPlannerException.class, + () -> getFragmentPlan("select inspect_mv_meta('a.b.c.d');")); + Assert.assertThrows(StarRocksPlannerException.class, + () -> getFragmentPlan("select inspect_mv_meta('db_notexists.mv1');")); + + // inspect_related_mv + testFragmentPlanContains("select inspect_related_mv('mv_base_table_9527')", "name\":\"mv1\""); + } + + @Test + public void testInspectHivePartitionInfo() throws Exception { + Assert.assertThrows(StarRocksPlannerException.class, + () -> testFragmentPlanContains("select inspect_hive_part_info('not_exist_catalog.no_db.no_table')", + "")); + testFragmentPlanContains("select inspect_hive_part_info('hive0.partitioned_db.lineitem_par')", "Project"); } @Test From 04777524f7aa6d787327753ec81961b05ef5e563 Mon Sep 17 00:00:00 2001 From: Murphy Date: Mon, 31 Jul 2023 11:35:05 +0800 Subject: [PATCH 03/23] fix conflict Signed-off-by: Murphy --- .../com/starrocks/analysis/TableName.java | 4 - .../starrocks/catalog/MaterializedView.java | 69 ------ .../com/starrocks/common/ErrorReport.java | 4 + .../com/starrocks/server/GlobalStateMgr.java | 4 +- .../com/starrocks/server/MetadataMgr.java | 67 ------ .../rewrite/ScalarOperatorFunctions.java | 198 +----------------- .../sql/plan/ConstantExpressionTest.java | 4 - 7 files changed, 11 insertions(+), 339 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java b/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java index da5142f1c65e51..9fda7b9f3d79fe 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/TableName.java @@ -36,11 +36,7 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.server.CatalogMgr; import com.starrocks.sql.analyzer.SemanticException; -<<<<<<< HEAD -======= -import com.starrocks.sql.parser.NodePosition; import org.apache.commons.lang3.StringUtils; ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) import java.io.DataInput; import java.io.DataOutput; diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java index 6d5189b4341bac..80f5565e040914 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java @@ -1191,76 +1191,7 @@ public void setPlanContext(MVRewriteContextCache mvRewriteContextCache) { this.mvRewriteContextCache = mvRewriteContextCache; } -<<<<<<< HEAD -======= - /** - * Infer the distribution info based on tables and MV query. - * Currently is max{bucket_num of base_table} - * TODO: infer the bucket number according to MV pattern and cardinality - */ - @Override - public void inferDistribution(DistributionInfo info) throws DdlException { - if (info.getBucketNum() == 0) { - int inferredBucketNum = 0; - for (BaseTableInfo base : getBaseTableInfos()) { - if (base.getTable().isNativeTableOrMaterializedView()) { - OlapTable olapTable = (OlapTable) base.getTable(); - DistributionInfo dist = olapTable.getDefaultDistributionInfo(); - inferredBucketNum = Math.max(inferredBucketNum, dist.getBucketNum()); - } - } - if (inferredBucketNum == 0) { - inferredBucketNum = CatalogUtils.calBucketNumAccordingToBackends(); - } - info.setBucketNum(inferredBucketNum); - } - } - - @Override - public Map getProperties() { - Map properties = super.getProperties(); - // For materialized view, add into session variables into properties. - if (super.getTableProperty() != null && super.getTableProperty().getProperties() != null) { - for (Map.Entry entry : super.getTableProperty().getProperties().entrySet()) { - if (entry.getKey().startsWith(PropertyAnalyzer.PROPERTIES_MATERIALIZED_VIEW_SESSION_PREFIX)) { - String varKey = entry.getKey().substring( - PropertyAnalyzer.PROPERTIES_MATERIALIZED_VIEW_SESSION_PREFIX.length()); - properties.put(varKey, entry.getValue()); - } - } - } - return properties; - } - - @Override - public void gsonPreProcess() throws IOException { - this.serializedPartitionRefTableExprs = new ArrayList<>(); - if (partitionRefTableExprs != null) { - for (Expr partitionExpr : partitionRefTableExprs) { - if (partitionExpr != null) { - serializedPartitionRefTableExprs.add( - new GsonUtils.ExpressionSerializedObject(partitionExpr.toSql())); - } - } - } - } - - @Override - public void gsonPostProcess() throws IOException { - super.gsonPostProcess(); - partitionRefTableExprs = new ArrayList<>(); - if (serializedPartitionRefTableExprs != null) { - for (GsonUtils.ExpressionSerializedObject expressionSql : serializedPartitionRefTableExprs) { - if (expressionSql != null) { - partitionRefTableExprs.add( - SqlParser.parseSqlToExpr(expressionSql.expressionSql, SqlModeHelper.MODE_DEFAULT)); - } - } - } - } - public String inspectMeta() { return GsonUtils.GSON.toJson(this); } ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/ErrorReport.java b/fe/fe-core/src/main/java/com/starrocks/common/ErrorReport.java index ddcfe80480adf3..f822cd0920eb0b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/ErrorReport.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/ErrorReport.java @@ -54,6 +54,10 @@ public static void reportAnalysisException(ErrorCode errorCode, Object... objs) reportAnalysisException(null, errorCode, objs); } + public static SemanticException buildSemanticException(ErrorCode errorCode, Object... objs) { + return new SemanticException(reportCommon(null, errorCode, objs)); + } + public static void reportSemanticException(ErrorCode errorCode, Object... objs) { reportSemanticException(null, errorCode, objs); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index 7112dd9da7dbfd..666b951d47fe67 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -284,6 +284,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -2646,8 +2647,6 @@ public Database getDb(String name) { return localMetastore.getDb(name); } -<<<<<<< HEAD -======= public Optional
mayGetTable(long dbId, long tableId) { return mayGetDb(dbId).flatMap(db -> db.tryGetTable(tableId)); } @@ -2660,7 +2659,6 @@ public Optional mayGetDb(long dbId) { return Optional.ofNullable(localMetastore.getDb(dbId)); } ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) public Database getDb(long dbId) { return localMetastore.getDb(dbId); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java index c997cb56a9a343..ecccffb7aeb756 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/MetadataMgr.java @@ -110,77 +110,10 @@ public List listTableNames(String catalogName, String dbName) { return ImmutableList.copyOf(tableNames.build()); } -<<<<<<< HEAD -======= - public boolean createTable(CreateTableStmt stmt) throws DdlException { - String catalogName = stmt.getCatalogName(); - Optional connectorMetadata = getOptionalMetadata(catalogName); - - if (connectorMetadata.isPresent()) { - if (!CatalogMgr.isInternalCatalog(catalogName)) { - String dbName = stmt.getDbName(); - String tableName = stmt.getTableName(); - if (getDb(catalogName, dbName) == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_BAD_DB_ERROR, dbName); - } - - if (listTableNames(catalogName, dbName).contains(tableName)) { - if (stmt.isSetIfNotExists()) { - LOG.info("create table[{}] which already exists", tableName); - return false; - } else { - ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); - } - } - } - return connectorMetadata.get().createTable(stmt); - } else { - throw new DdlException("Invalid catalog " + catalogName + " , ConnectorMetadata doesn't exist"); - } - } - - public void dropTable(String catalogName, String dbName, String tblName) { - TableName tableName = new TableName(catalogName, dbName, tblName); - DropTableStmt dropTableStmt = new DropTableStmt(false, tableName, false); - dropTable(dropTableStmt); - } - - public void dropTable(DropTableStmt stmt) { - String catalogName = stmt.getCatalogName(); - String dbName = stmt.getDbName(); - String tableName = stmt.getTableName(); - - Optional connectorMetadata = getOptionalMetadata(catalogName); - connectorMetadata.ifPresent(metadata -> { - try { - metadata.dropTable(stmt); - } catch (DdlException e) { - LOG.error("Failed to drop table {}.{}.{}", catalogName, dbName, tableName, e); - throw new StarRocksConnectorException("Failed to drop table {}.{}.{}", catalogName, dbName, tableName); - } - }); - } - public Optional
getTable(TableName tableName) { return Optional.ofNullable(getTable(tableName.getCatalog(), tableName.getDb(), tableName.getTbl())); } - public Table getTable(String catalogName, String dbName, String tblName) { - Optional connectorMetadata = getOptionalMetadata(catalogName); - Table connectorTable = connectorMetadata.map(metadata -> metadata.getTable(dbName, tblName)).orElse(null); - if (connectorTable != null) { - // Load meta information from ConnectorTblMetaInfoMgr for each external table. - connectorTblMetaInfoMgr.setTableInfoForConnectorTable(catalogName, dbName, connectorTable); - } - return connectorTable; - } - - public Pair getMaterializedViewIndex(String catalogName, String dbName, String tblName) { - Optional connectorMetadata = getOptionalMetadata(catalogName); - return connectorMetadata.map(metadata -> metadata.getMaterializedViewIndex(dbName, tblName)).orElse(null); - } - ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) public List listPartitionNames(String catalogName, String dbName, String tableName) { Optional connectorMetadata = getOptionalMetadata(catalogName); ImmutableSet.Builder partitionNames = ImmutableSet.builder(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java index 283f207f87ac1a..aa7590e27c4013 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java @@ -45,9 +45,11 @@ import com.starrocks.connector.PartitionInfo; import com.starrocks.connector.PartitionUtil; import com.starrocks.connector.hive.Partition; +import com.starrocks.mysql.privilege.PrivPredicate; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.analyzer.PrivilegeChecker; +import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.SetUtils; @@ -70,6 +72,7 @@ import java.time.format.ResolverStyle; import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAdjusters; +import java.util.Map; import java.util.Set; import static com.starrocks.catalog.PrimitiveType.BIGINT; @@ -96,39 +99,6 @@ public class ScalarOperatorFunctions { private static final Pattern HAS_TIME_PART = Pattern.compile("^.*[HhIiklrSsT]+.*$"); -<<<<<<< HEAD -======= - private static final int CONSTANT_128 = 128; - private static final BigInteger INT_128_OPENER = BigInteger.ONE.shiftLeft(CONSTANT_128 + 1); - private static final BigInteger[] INT_128_MASK1_ARR1 = new BigInteger[CONSTANT_128]; - - private static final int YEAR_MIN = 0; - private static final int YEAR_MAX = 9999; - private static final int DAY_OF_YEAR_MIN = 1; - private static final int DAY_OF_YEAR_MAX = 366; - - private static final LocalDateTime TIME_SLICE_START = LocalDateTime.of(1, 1, 1, 0, 0); - - private static final Map TIME_SLICE_UNIT_MAPPING; - - static { - for (int shiftBy = 0; shiftBy < CONSTANT_128; ++shiftBy) { - INT_128_MASK1_ARR1[shiftBy] = INT_128_OPENER.subtract(BigInteger.ONE).shiftRight(shiftBy + 1); - } - - TIME_SLICE_UNIT_MAPPING = ImmutableMap.builder() - .put("second", ChronoUnit.SECONDS) - .put("minute", ChronoUnit.MINUTES) - .put("hour", ChronoUnit.HOURS) - .put("day", ChronoUnit.DAYS) - .put("month", ChronoUnit.MONTHS) - .put("year", ChronoUnit.YEARS) - .put("week", ChronoUnit.WEEKS) - .put("quarter", IsoFields.QUARTER_YEARS) - .build(); - } - ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) /** * date and time function */ @@ -368,16 +338,6 @@ public static ConstantOperator fromUnixTime(ConstantOperator unixTime, ConstantO return dateFormat(dl, fmtLiteral); } -<<<<<<< HEAD - @ConstantFunction(name = "now", argTypes = {}, returnType = DATETIME) -======= - @ConstantFunction.List(list = { - @ConstantFunction(name = "now", argTypes = {}, returnType = DATETIME), - @ConstantFunction(name = "current_timestamp", argTypes = {}, returnType = DATETIME), - @ConstantFunction(name = "localtime", argTypes = {}, returnType = DATETIME), - @ConstantFunction(name = "localtimestamp", argTypes = {}, returnType = DATETIME) - }) ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) public static ConstantOperator now() { ConnectContext connectContext = ConnectContext.get(); LocalDateTime startTime = Instant.ofEpochMilli(connectContext.getStartTime()) @@ -425,145 +385,6 @@ public static ConstantOperator utcTimestamp() { return ConstantOperator.createDatetime(utcStartTime); } -<<<<<<< HEAD -======= - @ConstantFunction(name = "next_day", argTypes = {DATETIME, VARCHAR}, returnType = DATE) - public static ConstantOperator nextDay(ConstantOperator date, ConstantOperator dow) { - int dateDowValue = date.getDate().getDayOfWeek().getValue(); - switch (dow.getVarchar()) { - case "Sunday": - case "Sun": - case "Su": - return ConstantOperator.createDate(date.getDate().plusDays((13L - dateDowValue) % 7 + 1L)); - case "Monday": - case "Mon": - case "Mo": - return ConstantOperator.createDate(date.getDate().plusDays((7L - dateDowValue) % 7 + 1L)); - case "Tuesday": - case "Tue": - case "Tu": - return ConstantOperator.createDate(date.getDate().plusDays((8L - dateDowValue) % 7 + 1L)); - case "Wednesday": - case "Wed": - case "We": - return ConstantOperator.createDate(date.getDate().plusDays((9L - dateDowValue) % 7 + 1L)); - case "Thursday": - case "Thu": - case "Th": - return ConstantOperator.createDate(date.getDate().plusDays((10L - dateDowValue) % 7 + 1L)); - case "Friday": - case "Fri": - case "Fr": - return ConstantOperator.createDate(date.getDate().plusDays((11L - dateDowValue) % 7 + 1L)); - case "Saturday": - case "Sat": - case "Sa": - return ConstantOperator.createDate(date.getDate().plusDays((12L - dateDowValue) % 7 + 1L)); - default: - throw new IllegalArgumentException(dow + " not supported in next_day dow_string"); - } - } - - @ConstantFunction(name = "previous_day", argTypes = {DATETIME, VARCHAR}, returnType = DATE) - public static ConstantOperator previousDay(ConstantOperator date, ConstantOperator dow) { - int dateDowValue = date.getDate().getDayOfWeek().getValue(); - switch (dow.getVarchar()) { - case "Sunday": - case "Sun": - case "Su": - return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue - 1L) % 7 + 1L)); - case "Monday": - case "Mon": - case "Mo": - return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 5L) % 7 + 1L)); - case "Tuesday": - case "Tue": - case "Tu": - return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 4L) % 7 + 1L)); - case "Wednesday": - case "Wed": - case "We": - return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 3L) % 7 + 1L)); - case "Thursday": - case "Thu": - case "Th": - return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 2L) % 7 + 1L)); - case "Friday": - case "Fri": - case "Fr": - return ConstantOperator.createDate(date.getDate().minusDays((dateDowValue + 1L) % 7 + 1L)); - case "Saturday": - case "Sat": - case "Sa": - return ConstantOperator.createDate(date.getDate().minusDays(dateDowValue % 7 + 1L)); - default: - throw new IllegalArgumentException(dow + " not supported in previous_day dow_string"); - } - } - - @ConstantFunction(name = "makedate", argTypes = {INT, INT}, returnType = DATETIME) - public static ConstantOperator makeDate(ConstantOperator year, ConstantOperator dayOfYear) { - if (year.isNull() || dayOfYear.isNull()) { - return ConstantOperator.createNull(Type.DATE); - } - - int yearInt = year.getInt(); - if (yearInt < YEAR_MIN || yearInt > YEAR_MAX) { - return ConstantOperator.createNull(Type.DATE); - } - - int dayOfYearInt = dayOfYear.getInt(); - if (dayOfYearInt < DAY_OF_YEAR_MIN || dayOfYearInt > DAY_OF_YEAR_MAX) { - return ConstantOperator.createNull(Type.DATE); - } - - LocalDate ld = LocalDate.of(yearInt, 1, 1) - .plusDays(dayOfYearInt - 1); - - if (ld.getYear() != year.getInt()) { - return ConstantOperator.createNull(Type.DATE); - } - - return ConstantOperator.createDate(ld.atTime(0, 0, 0)); - } - - @ConstantFunction(name = "time_slice", argTypes = {DATETIME, INT, VARCHAR}, returnType = DATETIME) - public static ConstantOperator timeSlice(ConstantOperator datetime, ConstantOperator interval, - ConstantOperator unit) throws AnalysisException { - return timeSlice(datetime, interval, unit, ConstantOperator.createVarchar("floor")); - } - - @ConstantFunction(name = "time_slice", argTypes = {DATETIME, INT, VARCHAR, VARCHAR}, returnType = DATETIME) - public static ConstantOperator timeSlice(ConstantOperator datetime, ConstantOperator interval, - ConstantOperator unit, ConstantOperator boundary) - throws AnalysisException { - TemporalUnit timeUnit = TIME_SLICE_UNIT_MAPPING.get(unit.getVarchar()); - if (timeUnit == null) { - throw new IllegalArgumentException(unit + " not supported in time_slice unit param"); - } - boolean isEnd; - switch (boundary.getVarchar()) { - case "floor": - isEnd = false; - break; - case "ceil": - isEnd = true; - break; - default: - throw new IllegalArgumentException(boundary + " not supported in time_slice boundary param"); - } - long duration = TIME_SLICE_START.until(datetime.getDatetime(), timeUnit); - if (duration < 0) { - throw new AnalysisException("time used with time_slice can't before 0001-01-01 00:00:00"); - } - long epoch = duration - (duration % interval.getInt()); - if (isEnd) { - epoch += interval.getInt(); - } - return ConstantOperator.createDatetime(TIME_SLICE_START.plus(epoch, timeUnit)); - } - ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) /** * Math function */ @@ -1036,10 +857,7 @@ private static Table inspectExternalTable(TableName tableName) { Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(tableName) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName)); ConnectContext connectContext = ConnectContext.get(); - PrivilegeChecker.checkAnyActionOnTable( - connectContext.getCurrentUserIdentity(), - connectContext.getCurrentRoleIds(), - tableName); + PrivilegeChecker.checkTblPriv(connectContext, tableName, PrivPredicate.SELECT); return table; } @@ -1049,10 +867,7 @@ private static Pair inspectTable(TableName tableName) { Table table = db.tryGetTable(tableName.getTbl()) .orElseThrow(() -> ErrorReport.buildSemanticException(ErrorCode.ERR_BAD_TABLE_ERROR, tableName)); ConnectContext connectContext = ConnectContext.get(); - PrivilegeChecker.checkAnyActionOnTable( - connectContext.getCurrentUserIdentity(), - connectContext.getCurrentRoleIds(), - tableName); + PrivilegeChecker.checkTblPriv(connectContext, tableName, PrivPredicate.SELECT); return Pair.of(db, table); } @@ -1065,8 +880,7 @@ public static ConstantOperator inspect_mv_meta(ConstantOperator mvName) { Pair dbTable = inspectTable(tableName); Table table = dbTable.getRight(); if (!table.isMaterializedView()) { - ErrorReport.reportSemanticException(ErrorCode.ERR_INVALID_PARAMETER, - tableName + " is not materialized view"); + throw new SemanticException(tableName + " is not materialized view"); } try { dbTable.getLeft().readLock(); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java index ca0a3ba341f6e6..355a00a1fc671e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java @@ -17,11 +17,7 @@ package com.starrocks.sql.plan; -<<<<<<< HEAD -======= -import com.starrocks.qe.SqlModeHelper; import com.starrocks.sql.common.StarRocksPlannerException; ->>>>>>> 9eea14e87b ([Feature] support meta functions (#28094)) import com.starrocks.sql.parser.ParsingException; import org.junit.Assert; import org.junit.BeforeClass; From 19182c45225b8d14a7c39a9031a9012a17c7ccd9 Mon Sep 17 00:00:00 2001 From: Murphy Date: Tue, 1 Aug 2023 20:29:48 +0800 Subject: [PATCH 04/23] fix auth Signed-off-by: Murphy --- .../com/starrocks/sql/plan/ConstantExpressionTest.java | 8 +++++--- .../java/com/starrocks/sql/plan/PlanTestNoneDBBase.java | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java index 355a00a1fc671e..9e07ed77b8006a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java @@ -45,15 +45,17 @@ private void testFragmentPlanContains(String sql, String result) throws Exceptio public void testInspectMvMeta() throws Exception { String db = starRocksAssert.getCtx().getDatabase(); starRocksAssert.withTable( - "create table mv_base_table_9527 (id int, name string) properties('replication_num'='1')"); + "create table mv_base_table_9527 (id int, name string) " + + "distributed by hash(id) " + + "properties('replication_num'='1')"); starRocksAssert.withMaterializedView("create materialized view mv1 " + "distributed by hash(id) " + "refresh async " + "properties('replication_num'='1') " + "as select * from mv_base_table_9527"); - testFragmentPlanContains("select inspect_mv_meta('mv1');", "MaterializedView"); + testFragmentPlanContains("select inspect_mv_meta('mv1');", "refreshScheme"); String fullName = db + ".mv1"; - testFragmentPlanContains(String.format("select inspect_mv_meta('%s');", fullName), "MaterializedView"); + testFragmentPlanContains(String.format("select inspect_mv_meta('%s');", fullName), "refreshScheme"); // wrong arguments Assert.assertThrows(StarRocksPlannerException.class, diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java index 35dc6aa8578e70..8fcbdc4fbc766b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java @@ -21,6 +21,7 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.starrocks.analysis.UserIdentity; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; import com.starrocks.catalog.Table; @@ -81,7 +82,7 @@ public static void beforeClass() throws Exception { FeConstants.default_scheduler_interval_millisecond = 1; UtFrameUtils.createMinStarRocksCluster(); // create connect context - connectContext = UtFrameUtils.createDefaultCtx(); + connectContext = UtFrameUtils.initCtxForNewPrivilege(UserIdentity.ROOT); starRocksAssert = new StarRocksAssert(connectContext); connectContext.getSessionVariable().setOptimizerExecuteTimeout(30000); } From c198eed609e7547fa7dd4137edc19745ca9e92dd Mon Sep 17 00:00:00 2001 From: Murphy Date: Wed, 2 Aug 2023 13:41:40 +0800 Subject: [PATCH 05/23] fix ut Signed-off-by: Murphy --- .../test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java index 8fcbdc4fbc766b..35dc6aa8578e70 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PlanTestNoneDBBase.java @@ -21,7 +21,6 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.starrocks.analysis.UserIdentity; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; import com.starrocks.catalog.Table; @@ -82,7 +81,7 @@ public static void beforeClass() throws Exception { FeConstants.default_scheduler_interval_millisecond = 1; UtFrameUtils.createMinStarRocksCluster(); // create connect context - connectContext = UtFrameUtils.initCtxForNewPrivilege(UserIdentity.ROOT); + connectContext = UtFrameUtils.createDefaultCtx(); starRocksAssert = new StarRocksAssert(connectContext); connectContext.getSessionVariable().setOptimizerExecuteTimeout(30000); } From 94e400fefbbe272236b4f027fe3077d54b8ab7fb Mon Sep 17 00:00:00 2001 From: Murphy Date: Wed, 2 Aug 2023 14:11:42 +0800 Subject: [PATCH 06/23] fix ut Signed-off-by: Murphy --- .../MvRewriteOptimizationTest.java | 93 ------------------- 1 file changed, 93 deletions(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index 63dbcd8a9ee38e..7b33d91267f79f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -2723,99 +2723,6 @@ public void testJoinPredicatePushdown() throws Exception { PlanTestBase.assertContains(plan, "_pushdown_predicate_join_mv1"); } - @Test - public void testJoinPredicatePushdown2() throws Exception { - cluster.runSql("test", "CREATE TABLE `esrc__stays_reservations__335827106c571dc8400c45a1075473f1` (\n" + - " `reservation` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `guest` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `visitor_unique` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `host` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `listing` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `erf_guid` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `date` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `date_checkin_utc` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `date_checkout_utc` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `date_checkin_local` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `date_checkout_local` varchar(65533) NULL COMMENT \"Minerva Subject\",\n" + - " `ds` date NULL COMMENT \"Minerva Date Stamp\",\n" + - " `ts` datetime NULL COMMENT \"Minerva Time Stamp\",\n" + - " `dim_guest_count` varchar(65533) NULL COMMENT \"Reservation guest count, accounting for the cumulative changes to the reservation as of the time of the metric. In instances where a reservation is not altered, then this value will be equivalent to the value as of the booking time. For the guest count as of the time of the original booking, use dim_guest_count_at_booking.\",\n" + - " `dim_guest_count_categorical` varchar(65533) NULL COMMENT \"Reservation guest count, segmented, accounting for the cumulative changes to the reservation as of the time of the metric. In instances where a reservation is not altered, then this value will be equivalent to the value as of the booking time. For this dimension as of booking, use the dim_guest_count_categorical_at_booking. If m_guests = 0 (ie. no guests), this dimensions is equal to This value should not exist.\",\n" + - " `dim_trip_nights` varchar(65533) NULL COMMENT \"Number of nights in the reservation. May evolve over time due to alterations. For the number of nights as of the time of the booking, use dim_trip_nights_at_booking.\",\n" + - " `dim_trip_days_in_advance` varchar(65533) NULL COMMENT \"Number of days in advance between reservation event (i.e., booking, alteration, or cancellation) and trip start.\",\n" + - " `dim_trip_nights_long_stays` varchar(65533) NULL COMMENT \"Number of nights in the reservation, segmented for longer stays, accounting for the cumulative changes to the reservation as of the time of the metric. In instances where a reservation is not altered, then this value will be equivalent to the value as of the booking time.\",\n" + - " `net_bookings` double NULL COMMENT \"Minerva Measure\",\n" + - " `net_nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + - " `net_guest_nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + - " `bookings` double NULL COMMENT \"Minerva Measure\",\n" + - " `cancellations` double NULL COMMENT \"Minerva Measure\",\n" + - " `alterations` double NULL COMMENT \"Minerva Measure\",\n" + - " `nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + - " `nights_cancelled` double NULL COMMENT \"Minerva Measure\",\n" + - " `nights_altered` double NULL COMMENT \"Minerva Measure\",\n" + - " `guest_nights_booked` double NULL COMMENT \"Minerva Measure\",\n" + - " `guest_nights_cancelled` double NULL COMMENT \"Minerva Measure\",\n" + - " `guest_nights_altered` double NULL COMMENT \"Minerva Measure\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`reservation`, `guest`, `visitor_unique`, `host`, `listing`, `erf_guid`, `date`, `date_checkin_utc`, `date_checkout_utc`, `date_checkin_local`, `date_checkout_local`)\n" + - "COMMENT \"Minerva Airbnb Measure Source: stays_reservations\"\n" + - "PARTITION BY RANGE(`ds`)\n" + - "(PARTITION p20080807 VALUES [(\"0000-01-01\"), (\"2008-08-07\")),\n" + - "PARTITION p20230620 VALUES [(\"2023-05-21\"), (\"2023-06-20\")),\n" + - "PARTITION p_ds2023062020230719 VALUES [(\"2023-06-20\"), (\"2023-07-20\")))\n" + - "DISTRIBUTED BY HASH(`reservation`) BUCKETS 50\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"colocate_with\" = \"reservation_v5\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ");"); - cluster.runSql("test", "CREATE TABLE `dsrc__guest_age_groups__2b0ed629a2bbb51869caf8870198d587` (\n" + - " `guest` varchar(65533) NULL COMMENT \"\",\n" + - " `dim_guest_age_group` varchar(65533) NULL COMMENT \"\",\n" + - " `dim_guest_is_senior` varchar(65533) NULL COMMENT \"\",\n" + - " `dim_guest_generation_name` varchar(65533) NULL COMMENT \"\",\n" + - " `dim_guest_is_millennial` varchar(65533) NULL COMMENT \"\",\n" + - " `dim_guest_is_gen_x` varchar(65533) NULL COMMENT \"\",\n" + - " `dim_guest_is_boomer` varchar(65533) NULL COMMENT \"\",\n" + - " `ds` date NULL COMMENT \"\"\n" + - ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`guest`)\n" + - "DISTRIBUTED BY HASH(guest) \n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"in_memory\" = \"false\",\n" + - "\"enable_persistent_index\" = \"false\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"compression\" = \"LZ4\"\n" + - ");"); - createAndRefreshMv("test", "_hao_test_mv", "CREATE MATERIALIZED VIEW `_hao_test_mv` \n" + - "COMMENT \"MATERIALIZED_VIEW\"\n" + - "DISTRIBUTED BY HASH(`listing`) BUCKETS 18\n" + - "REFRESH MANUAL\n" + - "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + - "\"replicated_storage\" = \"true\",\n" + - "\"storage_medium\" = \"HDD\"\n" + - ")\n" + - "AS SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`listing`, `stays_reservations`.`host`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`, `stays_reservations`.`ts`, `stays_reservations`.`ds`, coalesce(`guest_age_groups`.`dim_guest_age_group`, '-unknown-') AS `dim_guest_age_group`, coalesce(`guest_age_groups`.`dim_guest_is_senior`, 'False') AS `dim_guest_is_senior`, coalesce(`guest_age_groups`.`dim_guest_generation_name`, '-unknown-') AS `dim_guest_generation_name`, coalesce(`guest_age_groups`.`dim_guest_is_millennial`, 'False') AS `dim_guest_is_millennial`, coalesce(`guest_age_groups`.`dim_guest_is_gen_x`, 'False') AS `dim_guest_is_gen_x`, coalesce(`guest_age_groups`.`dim_guest_is_boomer`, 'False') AS `dim_guest_is_boomer`, `guest_age_groups`.`ds` AS `guest_age_groups.ds`\n" + - "FROM (SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`host`, `stays_reservations`.`listing`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`ds`, `stays_reservations`.`ts`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`\n" + - "FROM `esrc__stays_reservations__335827106c571dc8400c45a1075473f1` AS `stays_reservations`) stays_reservations LEFT OUTER JOIN (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + - "FROM (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + - "FROM `dsrc__guest_age_groups__2b0ed629a2bbb51869caf8870198d587` AS `guest_age_groups`) guest_age_groups) guest_age_groups ON (((`stays_reservations`.`guest` IS NOT NULL) AND (`stays_reservations`.`guest` != '0')) AND (`stays_reservations`.`guest` = `guest_age_groups`.`guest`)) AND ((substring(CAST(`stays_reservations`.`ts` AS VARCHAR(65533)), 1, 10)) = `guest_age_groups`.`ds`);"); - - String query = - "select * from (SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`listing`, `stays_reservations`.`host`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`, `stays_reservations`.`ts`, `stays_reservations`.`ds`, coalesce(`guest_age_groups`.`dim_guest_age_group`, '-unknown-') AS `dim_guest_age_group`, coalesce(`guest_age_groups`.`dim_guest_is_senior`, 'False') AS `dim_guest_is_senior`, coalesce(`guest_age_groups`.`dim_guest_generation_name`, '-unknown-') AS `dim_guest_generation_name`, coalesce(`guest_age_groups`.`dim_guest_is_millennial`, 'False') AS `dim_guest_is_millennial`, coalesce(`guest_age_groups`.`dim_guest_is_gen_x`, 'False') AS `dim_guest_is_gen_x`, coalesce(`guest_age_groups`.`dim_guest_is_boomer`, 'False') AS `dim_guest_is_boomer`, `guest_age_groups`.`ds` AS `guest_age_groups.ds`\n" + - "FROM (SELECT `stays_reservations`.`reservation`, `stays_reservations`.`guest`, `stays_reservations`.`visitor_unique`, `stays_reservations`.`host`, `stays_reservations`.`listing`, `stays_reservations`.`erf_guid`, `stays_reservations`.`date`, `stays_reservations`.`date_checkin_utc`, `stays_reservations`.`date_checkout_utc`, `stays_reservations`.`date_checkin_local`, `stays_reservations`.`date_checkout_local`, `stays_reservations`.`ds`, `stays_reservations`.`ts`, `stays_reservations`.`dim_guest_count`, `stays_reservations`.`dim_guest_count_categorical`, `stays_reservations`.`dim_trip_nights`, `stays_reservations`.`dim_trip_days_in_advance`, `stays_reservations`.`dim_trip_nights_long_stays`, `stays_reservations`.`net_bookings`, `stays_reservations`.`net_nights_booked`, `stays_reservations`.`net_guest_nights_booked`, `stays_reservations`.`bookings`, `stays_reservations`.`cancellations`, `stays_reservations`.`alterations`, `stays_reservations`.`nights_booked`, `stays_reservations`.`nights_cancelled`, `stays_reservations`.`nights_altered`, `stays_reservations`.`guest_nights_booked`, `stays_reservations`.`guest_nights_cancelled`, `stays_reservations`.`guest_nights_altered`\n" + - "FROM `esrc__stays_reservations__335827106c571dc8400c45a1075473f1` AS `stays_reservations`) stays_reservations LEFT OUTER JOIN (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + - "FROM (SELECT `guest_age_groups`.`guest`, `guest_age_groups`.`dim_guest_age_group`, `guest_age_groups`.`dim_guest_is_senior`, `guest_age_groups`.`dim_guest_generation_name`, `guest_age_groups`.`dim_guest_is_millennial`, `guest_age_groups`.`dim_guest_is_gen_x`, `guest_age_groups`.`dim_guest_is_boomer`, `guest_age_groups`.`ds`\n" + - "FROM `dsrc__guest_age_groups__2b0ed629a2bbb51869caf8870198d587` AS `guest_age_groups`) guest_age_groups) guest_age_groups ON (((`stays_reservations`.`guest` IS NOT NULL) AND (`stays_reservations`.`guest` != '0')) AND (`stays_reservations`.`guest` = `guest_age_groups`.`guest`)) AND ((substring(CAST(`stays_reservations`.`ts` AS VARCHAR(65533)), 1, 10)) = `guest_age_groups`.`ds`)) _ limit 100;"; - String plan = getFragmentPlan(query); - PlanTestBase.assertContains(plan, "_hao_test_mv"); - } - @Ignore("outer join and pushdown predicate does not work") @Test public void testJoinPredicatePushdown1() throws Exception { From 8f911beb5b0d39152264938ad7d05413cdc3c5eb Mon Sep 17 00:00:00 2001 From: ABing <101158374+ABingHuang@users.noreply.github.com> Date: Wed, 2 Aug 2023 16:12:11 +0800 Subject: [PATCH 07/23] [BugFix][CherryPick] fix outer join and anti join rewrite bug for 2.5 (#28028) (#28466) Fixes #27163 The bug has the following causes: 1. The invalid use of equivalence in outer join and anti join, which leads to the invalid use of output columns. The join on predicate A.a = B.b for outer join and anti join can not be used to construct equivalence class. 2. the compatibility check should be more strict for outer and anti join, which should not consider the pushdown predicates derived from equivalence class. eg: the left outer join should make sure that the predicates of right child tree of query and mv should be equal. it means that the compensation predicates should not come from right child of left outer join. 3. when clone equivalence class, should use the same Set for the columns in the same ec. fixed by 1. constructing eq without column equal predicates in outer and anti join 2. add more strict compatibility check for outer and anti join, excluding pushdown ec derived predicates 3. fix equivalence class clone bug. Signed-off-by: ABingHuang --- .../optimizer/base/EquivalenceClasses.java | 7 +- .../operator/scalar/ScalarOperator.java | 12 ++ .../rewrite/JoinPredicatePushdown.java | 6 + .../optimizer/rule/mv/JoinDeriveContext.java | 17 +- .../MaterializedViewRewriter.java | 202 +++++++++++++++--- .../materialization/MvUtils.java | 33 ++- .../materialization/TableScanDesc.java | 38 +++- .../rule/BaseMaterializedViewRewriteRule.java | 20 +- .../planner/MaterializedViewTest.java | 112 ++++++---- .../sql/materialized-view/tpch-hive/q5.sql | 2 +- .../R/test_mv_join_derivabllity_rewrite | 49 ++++- .../T/test_mv_join_derivabllity_rewrite | 39 +++- 12 files changed, 422 insertions(+), 115 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/EquivalenceClasses.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/EquivalenceClasses.java index c2aa616f0329ac..6c9534a338f629 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/EquivalenceClasses.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/base/EquivalenceClasses.java @@ -25,7 +25,12 @@ public EquivalenceClasses clone() { final EquivalenceClasses ec = new EquivalenceClasses(); for (Map.Entry> entry : this.columnToEquivalenceClass.entrySet()) { - ec.columnToEquivalenceClass.put(entry.getKey(), Sets.newLinkedHashSet(entry.getValue())); + if (!ec.columnToEquivalenceClass.containsKey(entry.getKey())) { + Set columnEcs = Sets.newLinkedHashSet(entry.getValue()); + for (ColumnRefOperator column : columnEcs) { + ec.columnToEquivalenceClass.put(column, columnEcs); + } + } } ec.cacheColumnToEquivalenceClass = null; return ec; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ScalarOperator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ScalarOperator.java index 99c04bb14ef306..bf0872f712b45a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ScalarOperator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ScalarOperator.java @@ -25,6 +25,9 @@ public abstract class ScalarOperator implements Cloneable { // from JoinNode. protected boolean isRedundant = false; + // whether the ScalarOperator is pushdown from equivalence derivation + protected boolean isPushdown = false; + private List hints = Collections.emptyList(); public ScalarOperator(OperatorType opType, Type type) { @@ -141,6 +144,7 @@ public ScalarOperator clone() { operator = (ScalarOperator) super.clone(); operator.hints = Lists.newArrayList(hints); operator.isRedundant = this.isRedundant; + operator.isPushdown = this.isPushdown; } catch (CloneNotSupportedException ignored) { } return operator; @@ -209,6 +213,14 @@ public void setRedundant(boolean redundant) { isRedundant = redundant; } + public boolean isPushdown() { + return isPushdown; + } + + public void setIsPushdown(boolean isPushdown) { + this.isPushdown = isPushdown; + } + // whether ScalarOperator are equals without id public static boolean isEquivalent(ScalarOperator left, ScalarOperator right) { if (!left.getOpType().equals(right.getOpType())) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java index 41d65802e59ed4..32838e8dbd99e7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java @@ -391,6 +391,7 @@ private void getPushdownPredicatesFromEquivalenceDerive( if (join.getJoinType().isLeftSemiJoin()) { for (ScalarOperator p : derivedPredicates) { if (rightOutputColumns.containsAll(p.getUsedColumns())) { + p.setIsPushdown(true); rightPushDown.add(p); } } @@ -398,12 +399,14 @@ private void getPushdownPredicatesFromEquivalenceDerive( for (ScalarOperator p : derivedPredicates) { if (rightOutputColumns.containsAll(p.getUsedColumns()) && Utils.canEliminateNull(rightOutputColumnOps, p.clone())) { + p.setIsPushdown(true); rightPushDown.add(p); } } } else if (join.getJoinType().isRightSemiJoin()) { for (ScalarOperator p : derivedPredicates) { if (leftOutputColumns.containsAll(p.getUsedColumns())) { + p.setIsPushdown(true); leftPushDown.add(p); } } @@ -411,6 +414,7 @@ private void getPushdownPredicatesFromEquivalenceDerive( for (ScalarOperator p : derivedPredicates) { if (leftOutputColumns.containsAll(p.getUsedColumns()) && Utils.canEliminateNull(leftOutputColumnOps, p.clone())) { + p.setIsPushdown(true); leftPushDown.add(p); } } @@ -444,12 +448,14 @@ ScalarOperator equivalenceDeriveOnPredicate(ScalarOperator on, OptExpression joi } else if (join.getJoinType().isLeftOuterJoin()) { for (ScalarOperator p : derivedPredicates) { if (rightOutputColumns.containsAll(p.getUsedColumns())) { + p.setIsPushdown(true); pushDown.add(p); } } } else if (join.getJoinType().isRightOuterJoin()) { for (ScalarOperator p : derivedPredicates) { if (leftOutputColumns.containsAll(p.getUsedColumns())) { + p.setIsPushdown(true); pushDown.add(p); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java index 83d2be62070456..f37fe9fe52a4e3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java @@ -15,23 +15,28 @@ package com.starrocks.sql.optimizer.rule.mv; import com.starrocks.analysis.JoinOperator; +import com.starrocks.common.Pair; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import java.util.List; public class JoinDeriveContext { - private JoinOperator queryJoinType; - private JoinOperator mvJoinType; + private final JoinOperator queryJoinType; + private final JoinOperator mvJoinType; // join columns for left and right join tables - private List> joinColumns; + private final List> joinColumns; + + private final List> compensatedEquivalenceColumns; public JoinDeriveContext( JoinOperator queryJoinType, JoinOperator mvJoinType, - List> joinColumns) { + List> joinColumns, + List> compensatedEquivalenceColumns) { this.queryJoinType = queryJoinType; this.mvJoinType = mvJoinType; this.joinColumns = joinColumns; + this.compensatedEquivalenceColumns = compensatedEquivalenceColumns; } public JoinOperator getQueryJoinType() { @@ -49,4 +54,8 @@ public List getLeftJoinColumns() { public List getRightJoinColumns() { return joinColumns.get(1); } + + public List> getCompensatedEquivalenceColumns() { + return compensatedEquivalenceColumns; + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java index 518fa132560a83..e1ce676aaf5d3d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java @@ -185,7 +185,7 @@ private boolean isMVApplicable(OptExpression mvExpression, // query: a left outer join b // mv: a inner join b inner join c for (TableScanDesc queryScanDesc : queryTableScanDescs) { - if (queryScanDesc.getParentJoinType() != null + if (queryScanDesc.getJoinOptExpression() != null && !mvTableScanDescs.stream().anyMatch(scanDesc -> scanDesc.isMatch(queryScanDesc))) { logMVRewrite(mvRewriteContext, "MV is not applicable in view delta mode: " + "at least one same join type should be existed"); @@ -240,7 +240,12 @@ boolean computeCompatibility(OptExpression queryExpr, OptExpression mvExpr) { } if (queryJoinType.equals(mvJoinType)) { // it means both joins' type and onPredicate are equal - return true; + // for outer join, we should check some extra conditions + boolean ret = checkJoinMatch(queryJoinType, queryExpr, mvExpr); + if (!ret) { + logMVRewrite(mvRewriteContext, "join match check failed, joinType: %s", queryJoinType); + } + return ret; } if (!JOIN_COMPATIBLE_MAP.get(mvJoinType).contains(queryJoinType)) { @@ -253,29 +258,33 @@ boolean computeCompatibility(OptExpression queryExpr, OptExpression mvExpr) { // should know the right table ScalarOperator queryOnPredicate = queryJoin.getOnPredicate(); // relationId -> join on predicate used columns - Map joinColumns = Maps.newHashMap(); - boolean isSupported = - isSupportedPredicate(queryOnPredicate, materializationContext.getQueryRefFactory(), joinColumns); + Map tableToJoinColumns = Maps.newHashMap(); + List> joinColumnPairs = Lists.newArrayList(); + boolean isSupported = isSupportedPredicate(queryOnPredicate, + materializationContext.getQueryRefFactory(), tableToJoinColumns, joinColumnPairs); if (!isSupported) { logMVRewrite(mvRewriteContext, "join predicate is not supported %s", queryOnPredicate); return false; } // use join columns from query Map usedColumnsToTable = Maps.newHashMap(); - for (Map.Entry entry : joinColumns.entrySet()) { + for (Map.Entry entry : tableToJoinColumns.entrySet()) { Table table = materializationContext.getQueryRefFactory().getTableForColumn(entry.getValue().getFirstId()); usedColumnsToTable.put(entry.getValue(), table); } ColumnRefSet leftColumns = queryExpr.inputAt(0).getOutputColumns(); ColumnRefSet rightColumns = queryExpr.inputAt(1).getOutputColumns(); List> joinColumnRefs = Lists.newArrayList(Lists.newArrayList(), Lists.newArrayList()); - boolean isCompatible = - isJoinCompatible(usedColumnsToTable, queryJoinType, mvJoinType, leftColumns, rightColumns, joinColumnRefs); + // query based join columns pair + List> compensatedEquivalenceColumns = Lists.newArrayList(); + boolean isCompatible = isJoinCompatible(usedColumnsToTable, queryJoinType, mvJoinType, + leftColumns, rightColumns, joinColumnPairs, joinColumnRefs, compensatedEquivalenceColumns); if (!isCompatible) { logMVRewrite(mvRewriteContext, "join columns not compatible %s != %s", leftColumns, rightColumns); return false; } - JoinDeriveContext joinDeriveContext = new JoinDeriveContext(queryJoinType, mvJoinType, joinColumnRefs); + JoinDeriveContext joinDeriveContext = + new JoinDeriveContext(queryJoinType, mvJoinType, joinColumnRefs, compensatedEquivalenceColumns); mvRewriteContext.addJoinDeriveContext(joinDeriveContext); return true; } else if (queryOp instanceof LogicalScanOperator) { @@ -291,12 +300,64 @@ boolean computeCompatibility(OptExpression queryExpr, OptExpression mvExpr) { } } + private boolean checkJoinMatch(JoinOperator joinOperator, OptExpression queryExpr, OptExpression mvExpr) { + if (joinOperator.isInnerJoin() || joinOperator.isCrossJoin() || joinOperator.isSemiJoin()) { + return true; + } else if (joinOperator.isLeftOuterJoin() || joinOperator.isLeftAntiJoin()) { + // for left outer/anti join, we should make sure the predicates of right child should be equivalent + return checkJoinChildPredicate(queryExpr, mvExpr, 1); + } else if (joinOperator.isRightOuterJoin() || joinOperator.isRightAntiJoin()) { + // for right outer/anti join, we should make sure the predicates of left child should be equivalent + return checkJoinChildPredicate(queryExpr, mvExpr, 0); + } else if (joinOperator.isFullOuterJoin()) { + // for full outer join, we should make sure the predicates of both children should be equivalent + return checkJoinChildPredicate(queryExpr, mvExpr, 0) && checkJoinChildPredicate(queryExpr, mvExpr, 1); + } + return false; + } + + private boolean checkJoinChildPredicate(OptExpression queryExpr, OptExpression mvExpr, int index) { + Set queryPredicates = Sets.newHashSet(); + // extract all conjuncts + ScalarOperator normalizedQueryPredicate = + MvUtils.canonizePredicateForRewrite(Utils.compoundAnd(MvUtils.getAllPredicates(queryExpr.inputAt(index)))); + queryPredicates.addAll(Utils.extractConjuncts(normalizedQueryPredicate)); + queryPredicates = + queryPredicates.stream().filter(scalarOperator -> !scalarOperator.isPushdown()).collect(Collectors.toSet()); + Set mvPredicates = Sets.newHashSet(); + // extract all conjuncts + ScalarOperator normalizedMvPredicate = + MvUtils.canonizePredicateForRewrite(Utils.compoundAnd(MvUtils.getAllPredicates(mvExpr.inputAt(index)))); + mvPredicates.addAll(Utils.extractConjuncts(normalizedMvPredicate)); + mvPredicates = mvPredicates.stream().filter(scalarOperator -> !scalarOperator.isPushdown()).collect(Collectors.toSet()); + if (queryPredicates.isEmpty() && mvPredicates.isEmpty()) { + return true; + } + boolean isEqual = isAllPredicateEquivalent(queryPredicates, mvPredicates); + if (!isEqual) { + logMVRewrite( + mvRewriteContext, + "join child predicate not matched, queryPredicates: %s, mvPredicates: %s, index: %s", + queryPredicates, mvPredicates, index); + } + return isEqual; + } + + private boolean isAllPredicateEquivalent( + Collection queryPredicates, final Collection mvPredicates) { + return queryPredicates.size() == mvPredicates.size() + && queryPredicates.stream().allMatch(queryPredicate -> + mvPredicates.stream().anyMatch(mvPredicate -> mvPredicate.equivalent(queryPredicate))); + } + private boolean isJoinCompatible( Map usedColumnsToTable, JoinOperator queryJoinType, JoinOperator mvJoinType, ColumnRefSet leftColumns, ColumnRefSet rightColumns, - List> joinColumnRefs) { + List> joinColumnPairs, + List> joinColumnRefs, + List> compensatedEquivalenceColumns) { boolean isCompatible = true; if (mvJoinType.isInnerJoin() && queryJoinType.isLeftSemiJoin()) { // rewrite left semi join to inner join @@ -337,6 +398,10 @@ private boolean isJoinCompatible( List rightJoinColumnRefs = rightColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(1, rightJoinColumnRefs); + if (queryJoinType.isInnerJoin()) { + // for inner join, we should add the join columns into equivalence + compensatedEquivalenceColumns.addAll(joinColumnPairs); + } } else if (mvJoinType.isRightOuterJoin() && (queryJoinType.isInnerJoin() || queryJoinType.isRightAntiJoin())) { Optional leftColumnRefSet = usedColumnsToTable.keySet() .stream().filter(columnSet -> leftColumns.containsAll(columnSet)).findFirst(); @@ -346,6 +411,10 @@ private boolean isJoinCompatible( List leftJoinColumnRefs = leftColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(0, leftJoinColumnRefs); + if (queryJoinType.isInnerJoin()) { + // for inner join, we should add the join columns into equivalence + compensatedEquivalenceColumns.addAll(joinColumnPairs); + } } else if (mvJoinType.isFullOuterJoin() && queryJoinType.isLeftOuterJoin()) { Optional leftColumnRefSet = usedColumnsToTable.keySet() .stream().filter(columnSet -> leftColumns.containsAll(columnSet)).findFirst(); @@ -403,6 +472,8 @@ private boolean isJoinCompatible( List rightJoinColumnRefs = rightColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(1, rightJoinColumnRefs); + // for inner join, we should add the join columns into equivalence + compensatedEquivalenceColumns.addAll(joinColumnPairs); } return isCompatible; } @@ -421,7 +492,8 @@ private boolean isUniqueColumns(Table table, List columnNames) { } private boolean isSupportedPredicate( - ScalarOperator onPredicate, ColumnRefFactory columnRefFactory, Map joinColumns) { + ScalarOperator onPredicate, ColumnRefFactory columnRefFactory, Map tableToJoinColumns, + List> joinColumnPairs) { List conjuncts = Utils.extractConjuncts(onPredicate); List binaryPredicates = conjuncts.stream() .filter(conjunct -> ScalarOperator.isColumnEqualBinaryPredicate(conjunct)).collect(Collectors.toList()); @@ -429,6 +501,10 @@ private boolean isSupportedPredicate( return false; } + for (ScalarOperator scalarOperator : binaryPredicates) { + joinColumnPairs.add(Pair.create(scalarOperator.getChild(0).cast(), scalarOperator.getChild(1).cast())); + } + ColumnRefSet usedColumns = Utils.compoundAnd(binaryPredicates).getUsedColumns(); for (int columnId : usedColumns.getColumnIds()) { int relationId = columnRefFactory.getRelationId(columnId); @@ -436,10 +512,10 @@ private boolean isSupportedPredicate( // not use the column from table scan, unsupported return false; } - ColumnRefSet refColumns = joinColumns.computeIfAbsent(relationId, k -> new ColumnRefSet()); + ColumnRefSet refColumns = tableToJoinColumns.computeIfAbsent(relationId, k -> new ColumnRefSet()); refColumns.union(columnId); } - if (joinColumns.size() != 2) { + if (tableToJoinColumns.size() != 2) { // join predicate refs more than two tables, unsupported return false; } @@ -460,6 +536,7 @@ public OptExpression rewrite() { // Check whether mv can be applicable for the query. if (!isMVApplicable(mvExpression, queryTables, mvTables, matchMode, queryExpression)) { + logMVRewrite(mvRewriteContext, "mv applicable check failed"); return null; } @@ -550,13 +627,11 @@ private OptExpression rewriteViewDelta(List
queryTables, } mvDistinctScanDescs.add(mvExtraTableScanDescsSet); - final ScalarOperator mvEqualPredicate = mvPredicateSplit.getEqualPredicates(); - EquivalenceClasses viewEquivalenceClasses = createEquivalenceClasses(mvEqualPredicate); final Multimap compensationJoinColumns = ArrayListMultimap.create(); final Map> compensationRelations = Maps.newHashMap(); final Map expectedExtraQueryToMVRelationIds = Maps.newHashMap(); - if (!compensateViewDelta(viewEquivalenceClasses, mvTableScanDescs, mvExtraTableScanDescs, - compensationJoinColumns, compensationRelations, expectedExtraQueryToMVRelationIds, + if (!compensateViewDelta(mvTableScanDescs, mvExtraTableScanDescs, + compensationJoinColumns, compensationRelations, expectedExtraQueryToMVRelationIds, materializationContext)) { logMVRewrite(mvRewriteContext, "Rewrite ViewDelta failed: cannot compensate query by using PK/FK constraints"); continue; @@ -608,8 +683,8 @@ private OptExpression rewriteComplete(List
queryTables, .stream().flatMap(x -> x.values().stream()) .filter(columnRef -> !scanMvOutputColumns.contains(columnRef)) .collect(Collectors.toSet()); - final EquivalenceClasses queryEc = - createEquivalenceClasses(mvRewriteContext.getQueryPredicateSplit().getEqualPredicates()); + final EquivalenceClasses queryEc = createEquivalenceClasses( + mvRewriteContext.getQueryPredicateSplit().getEqualPredicates(), queryExpression, null); final RewriteContext rewriteContext = new RewriteContext( queryExpression, mvRewriteContext.getQueryPredicateSplit(), queryEc, queryRelationIdToColumns, materializationContext.getQueryRefFactory(), @@ -642,7 +717,16 @@ private OptExpression rewriteComplete(List
queryTables, // construct query based view EC final EquivalenceClasses queryBasedViewEqualPredicate = - createQueryBasedEquivalenceClasses(columnRewriter, mvEqualPredicate); + createEquivalenceClasses(mvEqualPredicate, mvExpression, columnRewriter); + if (queryBasedViewEqualPredicate == null) { + logMVRewrite(mvRewriteContext, "Rewrite complete failed: cannot construct query based equivalence classes"); + return null; + } + for (JoinDeriveContext deriveContext : mvRewriteContext.getJoinDeriveContexts()) { + for (Pair pair : deriveContext.getCompensatedEquivalenceColumns()) { + queryBasedViewEqualPredicate.addEquivalence(pair.first, pair.second); + } + } rewriteContext.setQueryBasedViewEquivalenceClasses(queryBasedViewEqualPredicate); OptExpression rewrittenExpression = tryRewriteForRelationMapping(rewriteContext, compensationJoinColumns); @@ -675,7 +759,6 @@ private OptExpression rewriteComplete(List
queryTables, // compensationJoinColumns and compensation table into compensationRelations. // return false if it can not be rewritten. private boolean compensateViewDelta( - EquivalenceClasses viewEquivalenceClasses, List mvTableScanDescs, List mvExtraTableScanDescs, Multimap compensationJoinColumns, @@ -742,21 +825,26 @@ private boolean compensateViewDelta( Multimap constraintCompensationJoinColumns = ArrayListMultimap.create(); if (!extraJoinCheck(mvParentTableScanDesc, mvTableScanDesc, columnPairs, childKeys, parentKeys, - viewEquivalenceClasses, constraintCompensationJoinColumns, materializedView)) { + constraintCompensationJoinColumns, materializedView)) { continue; } // If `mvParentTableScanDesc` is not included in query's plan, add it // to extraColumns. + JoinOperator joinOperator = mvParentTableScanDesc.getJoinType(); if (mvExtraTableScanDescs.contains(mvParentTableScanDesc)) { - compensationJoinColumns.putAll(constraintCompensationJoinColumns); + if (joinOperator.isInnerJoin() || joinOperator.isCrossJoin() || joinOperator.isSemiJoin()) { + compensationJoinColumns.putAll(constraintCompensationJoinColumns); + } List parentTableCompensationColumns = constraintCompensationJoinColumns.keys().stream().collect(Collectors.toList()); extraTableColumns.computeIfAbsent(mvParentTableScanDesc, x -> Lists.newArrayList()) .addAll(parentTableCompensationColumns); } if (mvExtraTableScanDescs.contains(mvTableScanDesc)) { - compensationJoinColumns.putAll(constraintCompensationJoinColumns); + if (joinOperator.isInnerJoin() || joinOperator.isCrossJoin() || joinOperator.isSemiJoin()) { + compensationJoinColumns.putAll(constraintCompensationJoinColumns); + } List childTableCompensationColumns = constraintCompensationJoinColumns.values().stream().collect(Collectors.toList()); extraTableColumns.computeIfAbsent(mvTableScanDesc, k -> Lists.newArrayList()) @@ -805,12 +893,14 @@ private boolean hasForeignKeyConstraintInMv(Table childTable, MaterializedView m private boolean extraJoinCheck( TableScanDesc parentTableScanDesc, TableScanDesc tableScanDesc, List> columnPairs, List childKeys, List parentKeys, - EquivalenceClasses viewEquivalenceClasses, Multimap constraintCompensationJoinColumns, MaterializedView materializedView) { Table parentTable = parentTableScanDesc.getTable(); Table childTable = tableScanDesc.getTable(); - JoinOperator parentJoinType = parentTableScanDesc.getParentJoinType(); + OptExpression joinOptExpr = parentTableScanDesc.getJoinOptExpression(); + Preconditions.checkNotNull(joinOptExpr); + LogicalJoinOperator joinOperator = joinOptExpr.getOp().cast(); + JoinOperator parentJoinType = joinOperator.getJoinType(); if (parentJoinType.isInnerJoin()) { // to check: // 1. childKeys should be foreign key @@ -843,15 +933,7 @@ private boolean extraJoinCheck( if (childColumn == null || parentColumn == null) { return false; } - if (viewEquivalenceClasses.getEquivalenceClass(childColumn) == null) { - return false; - } - if (viewEquivalenceClasses.getEquivalenceClass(parentColumn) == null) { - return false; - } - if (!viewEquivalenceClasses.getEquivalenceClass(childColumn).contains(parentColumn) - || !viewEquivalenceClasses.getEquivalenceClass(parentColumn).contains(childColumn)) { - // there is no join between childTable and parentTable + if (!isJoinOnCondition(joinOptExpr, childColumn, parentColumn)) { return false; } constraintCompensationJoinColumns.put(parentColumn, childColumn); @@ -859,6 +941,18 @@ private boolean extraJoinCheck( return true; } + private boolean isJoinOnCondition( + OptExpression joinExpr, + ColumnRefOperator childColumn, + ColumnRefOperator parentColumn) { + LogicalJoinOperator joinOperator = joinExpr.getOp().cast(); + List onConjuncts = Utils.extractConjuncts(joinOperator.getOnPredicate()); + List binaryConjuncts = + onConjuncts.stream().filter(ScalarOperator::isColumnEqualBinaryPredicate).collect(Collectors.toList()); + ColumnRefSet columns = new ColumnRefSet(Lists.newArrayList(childColumn, parentColumn)); + return binaryConjuncts.stream().anyMatch(conjunct -> conjunct.getUsedColumns().containsAll(columns)); + } + private boolean graphBasedCheck(MutableGraph graph, List extraTableScanDescs) { // remove one in-degree and zero out-degree's node from graph repeatedly @@ -1714,6 +1808,44 @@ protected EquationRewriter buildEquationRewriter( return rewriter; } + private EquivalenceClasses createEquivalenceClasses( + ScalarOperator equalPredicate, OptExpression expression, ColumnRewriter columnRewriter) { + List outerJoinOnPredicate = MvUtils.collectOuterAntiJoinOnPredicate(expression); + final PredicateSplit outerJoinPredicateSplit = PredicateSplit.splitPredicate(Utils.compoundAnd(outerJoinOnPredicate)); + return createEquivalenceClasses(equalPredicate, outerJoinPredicateSplit.getEqualPredicates(), columnRewriter); + } + + // equalPredicates eg: t1.a = t2.b and t1.c = t2.d + private EquivalenceClasses createEquivalenceClasses( + ScalarOperator equalPredicates, + ScalarOperator outerJoinEqualPredicates, + ColumnRewriter columnRewriter) { + EquivalenceClasses ec = new EquivalenceClasses(); + if (equalPredicates == null) { + return ec; + } + equalPredicates = MvUtils.canonizePredicateForRewrite(equalPredicates); + outerJoinEqualPredicates = MvUtils.canonizePredicateForRewrite(outerJoinEqualPredicates); + List outerJoinConjuncts = Utils.extractConjuncts(outerJoinEqualPredicates); + for (ScalarOperator equalPredicate : Utils.extractConjuncts(equalPredicates)) { + if (outerJoinConjuncts.contains(equalPredicate)) { + // outer join on predicates can not be used to construct equivalence class + continue; + } + Preconditions.checkState(equalPredicate.getChild(0).isColumnRef()); + ColumnRefOperator left = (ColumnRefOperator) equalPredicate.getChild(0); + Preconditions.checkState(equalPredicate.getChild(1).isColumnRef()); + ColumnRefOperator right = (ColumnRefOperator) equalPredicate.getChild(1); + ColumnRefOperator leftTarget = columnRewriter == null ? left : columnRewriter.rewriteViewToQuery(left).cast(); + ColumnRefOperator rightTarget = columnRewriter == null ? right : columnRewriter.rewriteViewToQuery(right).cast(); + if (leftTarget == null || rightTarget == null) { + return null; + } + ec.addEquivalence(leftTarget, rightTarget); + } + return ec; + } + // equalPredicates eg: t1.a = t2.b and t1.c = t2.d private EquivalenceClasses createEquivalenceClasses(ScalarOperator equalPredicates) { EquivalenceClasses ec = new EquivalenceClasses(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index 9231d702e808b8..86050637f119ae 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -168,7 +168,7 @@ public Void visitLogicalJoin(OptExpression optExpression, TableScanContext conte Integer id = scanContext.getTableIdMap().computeIfAbsent(table, t -> 0); LogicalJoinOperator joinOperator = optExpression.getOp().cast(); TableScanDesc tableScanDesc = new TableScanDesc( - table, id, scanOperator, joinOperator.getJoinType(), i == 0); + table, id, scanOperator, optExpression, i == 0); context.getTableScanDescs().add(tableScanDesc); scanContext.getTableIdMap().put(table, ++id); } else { @@ -412,8 +412,10 @@ public static ReplaceColumnRefRewriter getReplaceColumnRefWriter(OptExpression r return new ReplaceColumnRefRewriter(mvLineage, true); } + // pushdown predicates are excluded when calculating comppensation predicates, + // because they are derived from equivalence class, the original predicates have be considered private static void collectValidPredicates(List conjuncts, List predicates) { - conjuncts.stream().filter(x -> !x.isRedundant()).forEach(predicates::add); + conjuncts.stream().filter(x -> !x.isRedundant() && !x.isPushdown()).forEach(predicates::add); } private static void getAllPredicates(OptExpression root, List predicates) { @@ -861,4 +863,31 @@ public static String toString(Object o) { } return o.toString(); } + + public static List collectOnPredicate(OptExpression optExpression) { + List onPredicates = Lists.newArrayList(); + collectOnPredicate(optExpression, onPredicates, false); + return onPredicates; + } + + public static List collectOuterAntiJoinOnPredicate(OptExpression optExpression) { + List onPredicates = Lists.newArrayList(); + collectOnPredicate(optExpression, onPredicates, true); + return onPredicates; + } + + public static void collectOnPredicate( + OptExpression optExpression, List onPredicates, boolean onlyOuterAntiJoin) { + for (OptExpression child : optExpression.getInputs()) { + collectOnPredicate(child, onPredicates, onlyOuterAntiJoin); + } + if (optExpression.getOp() instanceof LogicalJoinOperator) { + LogicalJoinOperator joinOperator = optExpression.getOp().cast(); + if (onlyOuterAntiJoin && + !(joinOperator.getJoinType().isOuterJoin() || joinOperator.getJoinType().isAntiJoin())) { + return; + } + onPredicates.addAll(Utils.extractConjuncts(joinOperator.getOnPredicate())); + } + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/TableScanDesc.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/TableScanDesc.java index a105f9ad4229b7..94ce26ad704594 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/TableScanDesc.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/TableScanDesc.java @@ -17,6 +17,8 @@ import com.starrocks.analysis.JoinOperator; import com.starrocks.catalog.Table; +import com.starrocks.sql.optimizer.OptExpression; +import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; import java.util.Objects; @@ -27,16 +29,16 @@ public class TableScanDesc { private final int index; private final LogicalScanOperator scanOperator; // join type of LogicalJoinOperator above scan operator - private final JoinOperator parentJoinType; + private final OptExpression joinOptExpression; private final boolean isLeft; public TableScanDesc(Table table, int index, - LogicalScanOperator scanOperator, JoinOperator parentJoinType, + LogicalScanOperator scanOperator, OptExpression joinOptExpression, boolean isLeft) { this.table = table; this.index = index; this.scanOperator = scanOperator; - this.parentJoinType = parentJoinType; + this.joinOptExpression = joinOptExpression; this.isLeft = isLeft; } @@ -48,8 +50,8 @@ public int getIndex() { return index; } - public JoinOperator getParentJoinType() { - return parentJoinType; + public OptExpression getJoinOptExpression() { + return joinOptExpression; } public String getName() { @@ -60,6 +62,14 @@ public LogicalScanOperator getScanOperator() { return scanOperator; } + public JoinOperator getJoinType() { + if (joinOptExpression == null) { + return null; + } + LogicalJoinOperator joinOperator = joinOptExpression.getOp().cast(); + return joinOperator.getJoinType(); + } + public boolean isMatch(TableScanDesc other) { boolean matched = table.equals(other.table); if (!matched) { @@ -69,16 +79,24 @@ public boolean isMatch(TableScanDesc other) { // for // query: a left join c // mv: a inner join b left join c - if (parentJoinType.isInnerJoin()) { - return other.parentJoinType.isInnerJoin() || (other.parentJoinType.isLeftOuterJoin() && other.isLeft); + JoinOperator joinOperator = getJoinType(); + JoinOperator otherJoinOperator = other.getJoinType(); + if (joinOperator == null && otherJoinOperator == null) { + return true; + } else if (joinOperator == null || otherJoinOperator == null) { + return false; + } + if (joinOperator.isInnerJoin()) { + return otherJoinOperator.isInnerJoin() + || (otherJoinOperator.isLeftOuterJoin() && other.isLeft); } // for // query: a inner join c // mv: a left outer join b inner join c - if (parentJoinType.isLeftOuterJoin()) { - return (isLeft && other.parentJoinType.isInnerJoin()) - || (other.parentJoinType.isLeftOuterJoin() && isLeft == other.isLeft); + if (joinOperator.isLeftOuterJoin()) { + return (isLeft && otherJoinOperator.isInnerJoin()) + || (otherJoinOperator.isLeftOuterJoin() && isLeft == other.isLeft); } return false; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java index 763fb82a1bcae2..e6b3d37184fb7c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java @@ -10,7 +10,6 @@ import com.starrocks.sql.optimizer.OptimizerContext; import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.base.ColumnRefFactory; -import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.pattern.Pattern; import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; @@ -86,7 +85,7 @@ public List transform(OptExpression queryExpression, OptimizerCon queryPredicate = MvUtils.canonizePredicate(Utils.compoundAnd(queryPredicate, queryPartitionPredicate)); } final PredicateSplit queryPredicateSplit = PredicateSplit.splitPredicate(queryPredicate); - List onPredicates = collectOnPredicate(queryExpression); + List onPredicates = MvUtils.collectOnPredicate(queryExpression); onPredicates = onPredicates.stream().map(MvUtils::canonizePredicate).collect(Collectors.toList()); List
queryTables = MvUtils.getAllTables(queryExpression); for (MaterializationContext mvContext : mvCandidateContexts) { @@ -110,23 +109,6 @@ public List transform(OptExpression queryExpression, OptimizerCon return results; } - private List collectOnPredicate(OptExpression optExpression) { - List onPredicates = Lists.newArrayList(); - collectOnPredicate(optExpression, onPredicates); - return onPredicates; - } - - private void collectOnPredicate(OptExpression optExpression, List onPredicates) { - if (optExpression.getOp() instanceof LogicalJoinOperator) { - LogicalJoinOperator joinOperator = optExpression.getOp().cast(); - onPredicates.addAll(Utils.extractConjuncts(joinOperator.getOnPredicate())); - } else { - for (OptExpression child : optExpression.getInputs()) { - collectOnPredicate(child, onPredicates); - } - } - } - /** * After plan is rewritten by MV, still do some actions for new MV's plan. * 1. column prune diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java index 7dc9df358a7d18..2b11f894fd2aa6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java @@ -519,6 +519,7 @@ public void testSwapOuterJoin() { public void testLeftOuterJoinQueryComplete() { String mv = "select deptno as col1, empid as col2, emps.locationid as col3 from emps " + " left join locations on emps.locationid = locations.locationid"; + testRewriteOK(mv, "select count(*) from " + "emps left join locations on emps.locationid = locations.locationid"); testRewriteOK(mv, "select empid as col2, emps.locationid from " + @@ -527,7 +528,8 @@ public void testLeftOuterJoinQueryComplete() { testRewriteOK(mv, "select count(*) from " + "emps left join locations on emps.locationid = locations.locationid " + "where emps.deptno > 10"); - testRewriteOK(mv, "select empid as col2, locations.locationid from " + + + testRewriteOK(mv, "select empid as col2, emps.locationid from " + "emps left join locations on emps.locationid = locations.locationid " + "where emps.locationid > 10"); // TODO: Query's left outer join will be converted to Inner Join. @@ -556,7 +558,7 @@ public void testRightOuterJoinQueryComplete() { testRewriteOK(mv, "select count(*) from " + "emps right join locations on emps.locationid = locations.locationid " + "where emps.deptno > 10"); - testRewriteOK(mv, "select empid as col2, locations.locationid from " + + testRewriteFail(mv, "select empid as col2, locations.locationid from " + "emps right join locations on emps.locationid = locations.locationid " + "where locations.locationid > 10"); testRewriteOK(mv, "select empid as col2, locations.locationid from " + @@ -1702,48 +1704,49 @@ public void testInnerJoinViewDelta() { @Test public void testOuterJoinViewDelta() { - connectContext.getSessionVariable().setOptimizerExecuteTimeout(300000000); - String mv = "SELECT" + - " `l`.`LO_ORDERKEY` as col1, `l`.`LO_ORDERDATE`, `l`.`LO_LINENUMBER`, `l`.`LO_CUSTKEY`, `l`.`LO_PARTKEY`," + - " `l`.`LO_SUPPKEY`, `l`.`LO_ORDERPRIORITY`, `l`.`LO_SHIPPRIORITY`, `l`.`LO_QUANTITY`," + - " `l`.`LO_EXTENDEDPRICE`, `l`.`LO_ORDTOTALPRICE`, `l`.`LO_DISCOUNT`, `l`.`LO_REVENUE`," + - " `l`.`LO_SUPPLYCOST`, `l`.`LO_TAX`, `l`.`LO_COMMITDATE`, `l`.`LO_SHIPMODE`," + - " `c`.`C_NAME`, `c`.`C_ADDRESS`, `c`.`C_CITY`, `c`.`C_NATION`, `c`.`C_REGION`, `c`.`C_PHONE`," + - " `c`.`C_MKTSEGMENT`, `s`.`S_NAME`, `s`.`S_ADDRESS`, `s`.`S_CITY`, `s`.`S_NATION`, `s`.`S_REGION`," + - " `s`.`S_PHONE`, `p`.`P_NAME`, `p`.`P_MFGR`, `p`.`P_CATEGORY`, `p`.`P_BRAND`, `p`.`P_COLOR`," + - " `p`.`P_TYPE`, `p`.`P_SIZE`, `p`.`P_CONTAINER`\n" + - "FROM `lineorder` AS `l` " + - " LEFT OUTER JOIN `customer` AS `c` ON `c`.`C_CUSTKEY` = `l`.`LO_CUSTKEY`" + - " LEFT OUTER JOIN `supplier` AS `s` ON `s`.`S_SUPPKEY` = `l`.`LO_SUPPKEY`" + - " LEFT OUTER JOIN `part` AS `p` ON `p`.`P_PARTKEY` = `l`.`LO_PARTKEY`;"; - - String query = - "SELECT `lineorder`.`lo_orderkey`, `lineorder`.`lo_orderdate`, `customer`.`c_custkey` AS `cd`\n" + - "FROM `lineorder` LEFT OUTER JOIN `customer` ON `lineorder`.`lo_custkey` = `customer`.`c_custkey`\n" + - "WHERE `lineorder`.`lo_orderkey` = 100;"; + { + String mv = "SELECT" + + " `l`.`LO_ORDERKEY` as col1, `l`.`LO_ORDERDATE`, `l`.`LO_LINENUMBER`, `l`.`LO_CUSTKEY`, `l`.`LO_PARTKEY`," + + " `l`.`LO_SUPPKEY`, `l`.`LO_ORDERPRIORITY`, `l`.`LO_SHIPPRIORITY`, `l`.`LO_QUANTITY`," + + " `l`.`LO_EXTENDEDPRICE`, `l`.`LO_ORDTOTALPRICE`, `l`.`LO_DISCOUNT`, `l`.`LO_REVENUE`," + + " `l`.`LO_SUPPLYCOST`, `l`.`LO_TAX`, `l`.`LO_COMMITDATE`, `l`.`LO_SHIPMODE`," + + " `c`.`C_NAME`, `c`.`C_ADDRESS`, `c`.`C_CITY`, `c`.`C_NATION`, `c`.`C_REGION`, `c`.`C_PHONE`," + + " `c`.`C_MKTSEGMENT`, `s`.`S_NAME`, `s`.`S_ADDRESS`, `s`.`S_CITY`, `s`.`S_NATION`, `s`.`S_REGION`," + + " `s`.`S_PHONE`, `p`.`P_NAME`, `p`.`P_MFGR`, `p`.`P_CATEGORY`, `p`.`P_BRAND`, `p`.`P_COLOR`," + + " `p`.`P_TYPE`, `p`.`P_SIZE`, `p`.`P_CONTAINER`\n" + + "FROM `lineorder` AS `l` " + + " LEFT OUTER JOIN `customer` AS `c` ON `c`.`C_CUSTKEY` = `l`.`LO_CUSTKEY`" + + " LEFT OUTER JOIN `supplier` AS `s` ON `s`.`S_SUPPKEY` = `l`.`LO_SUPPKEY`" + + " LEFT OUTER JOIN `part` AS `p` ON `p`.`P_PARTKEY` = `l`.`LO_PARTKEY`;"; + + String query = "SELECT `lineorder`.`lo_orderkey`, `lineorder`.`lo_orderdate`, `lineorder`.`lo_custkey` AS `cd`\n" + + "FROM `lineorder` LEFT OUTER JOIN `customer` ON `lineorder`.`lo_custkey` = `customer`.`c_custkey`\n" + + "WHERE `lineorder`.`lo_orderkey` = 100;"; - testRewriteOK(mv, query); + testRewriteOK(mv, query); + } - String mv2 = "SELECT" + - " `l`.`LO_ORDERKEY` as col1, `l`.`LO_ORDERDATE`, `l`.`LO_LINENUMBER`, `l`.`LO_CUSTKEY`, `l`.`LO_PARTKEY`," + - " `l`.`LO_SUPPKEY`, `l`.`LO_ORDERPRIORITY`, `l`.`LO_SHIPPRIORITY`, `l`.`LO_QUANTITY`," + - " `l`.`LO_EXTENDEDPRICE`, `l`.`LO_ORDTOTALPRICE`, `l`.`LO_DISCOUNT`, `l`.`LO_REVENUE`," + - " `l`.`LO_SUPPLYCOST`, `l`.`LO_TAX`, `l`.`LO_COMMITDATE`, `l`.`LO_SHIPMODE`," + - " `c`.`C_NAME`, `c`.`C_ADDRESS`, `c`.`C_CITY`, `c`.`C_NATION`, `c`.`C_REGION`, `c`.`C_PHONE`," + - " `c`.`C_MKTSEGMENT`, `s`.`S_NAME`, `s`.`S_ADDRESS`, `s`.`S_CITY`, `s`.`S_NATION`, `s`.`S_REGION`," + - " `s`.`S_PHONE`, `p`.`P_NAME`, `p`.`P_MFGR`, `p`.`P_CATEGORY`, `p`.`P_BRAND`, `p`.`P_COLOR`," + - " `p`.`P_TYPE`, `p`.`P_SIZE`, `p`.`P_CONTAINER`\n" + - "FROM `lineorder_null` AS `l` " + - " LEFT OUTER JOIN `customer` AS `c` ON `c`.`C_CUSTKEY` = `l`.`LO_CUSTKEY`" + - " LEFT OUTER JOIN `supplier` AS `s` ON `s`.`S_SUPPKEY` = `l`.`LO_SUPPKEY`" + - " LEFT OUTER JOIN `part` AS `p` ON `p`.`P_PARTKEY` = `l`.`LO_PARTKEY`;"; + { + String mv2 = "SELECT" + + " `l`.`LO_ORDERKEY` as col1, `l`.`LO_ORDERDATE`, `l`.`LO_LINENUMBER`, `l`.`LO_CUSTKEY`, `l`.`LO_PARTKEY`," + + " `l`.`LO_SUPPKEY`, `l`.`LO_ORDERPRIORITY`, `l`.`LO_SHIPPRIORITY`, `l`.`LO_QUANTITY`," + + " `l`.`LO_EXTENDEDPRICE`, `l`.`LO_ORDTOTALPRICE`, `l`.`LO_DISCOUNT`, `l`.`LO_REVENUE`," + + " `l`.`LO_SUPPLYCOST`, `l`.`LO_TAX`, `l`.`LO_COMMITDATE`, `l`.`LO_SHIPMODE`," + + " `c`.`C_NAME`, `c`.`C_ADDRESS`, `c`.`C_CITY`, `c`.`C_NATION`, `c`.`C_REGION`, `c`.`C_PHONE`," + + " `c`.`C_MKTSEGMENT`, `s`.`S_NAME`, `s`.`S_ADDRESS`, `s`.`S_CITY`, `s`.`S_NATION`, `s`.`S_REGION`," + + " `s`.`S_PHONE`, `p`.`P_NAME`, `p`.`P_MFGR`, `p`.`P_CATEGORY`, `p`.`P_BRAND`, `p`.`P_COLOR`," + + " `p`.`P_TYPE`, `p`.`P_SIZE`, `p`.`P_CONTAINER`\n" + + "FROM `lineorder_null` AS `l` " + + " LEFT OUTER JOIN `customer` AS `c` ON `c`.`C_CUSTKEY` = `l`.`LO_CUSTKEY`" + + " LEFT OUTER JOIN `supplier` AS `s` ON `s`.`S_SUPPKEY` = `l`.`LO_SUPPKEY`" + + " LEFT OUTER JOIN `part` AS `p` ON `p`.`P_PARTKEY` = `l`.`LO_PARTKEY`;"; - String query2 = - "SELECT `lineorder_null`.`lo_orderkey`, `lineorder_null`.`lo_orderdate`, `customer`.`c_custkey` AS `cd`\n" + - "FROM `lineorder_null` LEFT OUTER JOIN `customer` ON `lineorder_null`.`lo_custkey` = `customer`.`c_custkey`\n" + - "WHERE `lineorder_null`.`lo_orderkey` = 100;"; + String query2 = "SELECT `lineorder_null`.`lo_orderkey`, `lineorder_null`.`lo_orderdate`, `lineorder_null`.`lo_custkey` AS `cd`\n" + + "FROM `lineorder_null` LEFT OUTER JOIN `customer` ON `lineorder_null`.`lo_custkey` = `customer`.`c_custkey`\n" + + "WHERE `lineorder_null`.`lo_orderkey` = 100;"; - testRewriteOK(mv2, query2); + testRewriteOK(mv2, query2); + } } @Test @@ -3123,7 +3126,8 @@ public void testCountDistinctRollupAgg() throws Exception { String mv = "select lo_orderkey, lo_linenumber, lo_quantity, lo_revenue, c_custkey, c_name" + " from lineorder left outer join customer" + " on lo_custkey = c_custkey"; - String query = "select lo_orderkey, lo_linenumber, lo_quantity, lo_revenue, lo_custkey" + + + String query = "select lo_orderkey, lo_linenumber, lo_quantity, lo_revenue" + " from lineorder left anti join customer" + " on lo_custkey = c_custkey"; MVRewriteChecker checker = testRewriteOK(mv, query); @@ -3728,4 +3732,30 @@ public void testJoinWithTypeCast() throws Exception { " GROUP BY `col1_name`;") .match("test_mv1"); } + + @Test + public void testJoinDerive() throws Exception { + starRocksAssert.withTable("CREATE TABLE t5 (\n" + + " k1 int,\n" + + " k2 int not null\n" + + " )\n" + + " DUPLICATE KEY(k1) " + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 12\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"in_memory\" = \"false\"\n" + + ")\n"); + starRocksAssert.withTable("CREATE TABLE t4 (\n" + + " a int,\n" + + " b int not null\n" + + " )\n" + + " DUPLICATE KEY(a) " + + "DISTRIBUTED BY HASH(`a`) BUCKETS 12\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"in_memory\" = \"false\"\n" + + ")\n"); + testRewriteOK("select * from t5 full outer join t4 on k1=a", + "select * from t5 left outer join t4 on k1=a where k1=3;"); + } } diff --git a/fe/fe-core/src/test/resources/sql/materialized-view/tpch-hive/q5.sql b/fe/fe-core/src/test/resources/sql/materialized-view/tpch-hive/q5.sql index e709df83669d0b..69f737db391864 100644 --- a/fe/fe-core/src/test/resources/sql/materialized-view/tpch-hive/q5.sql +++ b/fe/fe-core/src/test/resources/sql/materialized-view/tpch-hive/q5.sql @@ -29,6 +29,6 @@ TOP-N (order by [[49: sum DESC NULLS LAST]]) AGGREGATE ([GLOBAL] aggregate [{49: sum=sum(49: sum)}] group by [[42: n_name]] having [null] EXCHANGE SHUFFLE[42] AGGREGATE ([LOCAL] aggregate [{49: sum=sum(48: expr)}] group by [[42: n_name]] having [null] - SCAN (mv[lineitem_mv] columns[70: c_nationkey, 85: o_orderdate, 96: s_nationkey, 98: l_saleprice, 104: n_name2, 107: r_name2] predicate[96: s_nationkey = 70: c_nationkey AND 85: o_orderdate >= 1995-01-01 AND 85: o_orderdate < 1996-01-01 AND 107: r_name2 = AFRICA]) + SCAN (mv[lineitem_mv] columns[75: c_nationkey, 90: o_orderdate, 101: s_nationkey, 103: l_saleprice, 109: n_name2, 112: r_name2] predicate[75: c_nationkey = 101: s_nationkey AND 90: o_orderdate >= 1995-01-01 AND 90: o_orderdate < 1996-01-01 AND 112: r_name2 = AFRICA]) [end] diff --git a/test/sql/test_materialized_view/R/test_mv_join_derivabllity_rewrite b/test/sql/test_materialized_view/R/test_mv_join_derivabllity_rewrite index 82f7831ab19855..36d63049add503 100644 --- a/test/sql/test_materialized_view/R/test_mv_join_derivabllity_rewrite +++ b/test/sql/test_materialized_view/R/test_mv_join_derivabllity_rewrite @@ -617,4 +617,51 @@ order by lo_orderkey, lo_linenumber; 10002 1 10 2000 2 name_2 10002 2 10 2500 2 name_2 10003 1 30 3000 3 name_3 -10003 1 30 3600 3 name_3 \ No newline at end of file +10003 1 30 3600 3 name_3 + + +-- name: test_outer_join_rewrite + +CREATE TABLE t1 ( + k1 int, + k2 int not null + ) + DUPLICATE KEY(k1); +CREATE TABLE t2 ( + a int, + b int not null + ) + DUPLICATE KEY(a); +INSERT INTO t1 VALUES (1,1),(3,2),(null,1); +INSERT INTO t2 VALUES (1,1),(2,2),(null,1); +CREATE MATERIALIZED VIEW mv1 REFRESH MANUAL AS select * from t1 full outer join t2 on k1=a; +REFRESH MATERIALIZED VIEW mv1 with sync mode; +select * from t1 left outer join t2 on k1=a where k1=3; +-- result: +3 2 NULL NULL + +select * from t1 right outer join t2 on k1=a where b=2; +-- result: +NULL NULL 2 2 + +-- name: test_anti_join_rewrite + +CREATE TABLE t1 ( + k1 int, + k2 int + ) + DUPLICATE KEY(k1); +CREATE TABLE t2 ( + a int, + b int + ) + DUPLICATE KEY(a); +INSERT INTO t1 VALUES (1,1),(3,2),(1,null),(null,null); +INSERT INTO t2 VALUES (1,1),(2,2),(null,1),(null,null); +CREATE MATERIALIZED VIEW mv1 REFRESH MANUAL AS select * from t1 right outer join t2 on k1=a; +REFRESH MATERIALIZED VIEW mv1 with sync mode; +select * from t1 right anti join t2 on k1=a order by 1,2; +-- result: +NULL NULL +NULL 1 +2 2 \ No newline at end of file diff --git a/test/sql/test_materialized_view/T/test_mv_join_derivabllity_rewrite b/test/sql/test_materialized_view/T/test_mv_join_derivabllity_rewrite index 83acedc84de8e2..ae3d6464d6ed5d 100644 --- a/test/sql/test_materialized_view/T/test_mv_join_derivabllity_rewrite +++ b/test/sql/test_materialized_view/T/test_mv_join_derivabllity_rewrite @@ -430,4 +430,41 @@ on lo_custkey = c_custkey; select lo_orderkey, lo_linenumber, lo_quantity, lo_custkey, c_custkey, c_name from lineorder inner join customer on lo_custkey = c_custkey -order by lo_orderkey, lo_linenumber; \ No newline at end of file +order by lo_orderkey, lo_linenumber; + +-- name: test_outer_join_rewrite + +CREATE TABLE t1 ( + k1 int, + k2 int not null + ) + DUPLICATE KEY(k1); +CREATE TABLE t2 ( + a int, + b int not null + ) + DUPLICATE KEY(a); +INSERT INTO t1 VALUES (1,1),(3,2),(null,1); +INSERT INTO t2 VALUES (1,1),(2,2),(null,1); +CREATE MATERIALIZED VIEW mv1 REFRESH MANUAL AS select * from t1 full outer join t2 on k1=a; +REFRESH MATERIALIZED VIEW mv1 with sync mode; +select * from t1 left outer join t2 on k1=a where k1=3; +select * from t1 right outer join t2 on k1=a where b=2; + +-- name: test_anti_join_rewrite + +CREATE TABLE t1 ( + k1 int, + k2 int + ) + DUPLICATE KEY(k1); +CREATE TABLE t2 ( + a int, + b int + ) + DUPLICATE KEY(a); +INSERT INTO t1 VALUES (1,1),(3,2),(1,null),(null,null); +INSERT INTO t2 VALUES (1,1),(2,2),(null,1),(null,null); +CREATE MATERIALIZED VIEW mv1 REFRESH MANUAL AS select * from t1 right outer join t2 on k1=a; +REFRESH MATERIALIZED VIEW mv1 with sync mode; +select * from t1 right anti join t2 on k1=a order by 1,2; \ No newline at end of file From 3716db29ec364ad2d163b701b6bac46f47289fbd Mon Sep 17 00:00:00 2001 From: trueeyu Date: Tue, 1 Aug 2023 10:33:19 +0800 Subject: [PATCH 08/23] [Refactor] Change the log level of varchar length exceed limit (#28297) Fixes #issue Change log level to VLOG Before change: ``` W0731 15:38:23.014871 100302 string_converter.cpp:62] Column [binary]'s length exceeds max varchar length. ``` After change: ``` I0731 16:05:00.604197 84989 string_converter.cpp:63] Column [binary]'s length exceeds max varchar length. str_size(9), max_size(5) ``` (cherry picked from commit 7b862577c3fc1d3ecec7060362bde85c1af09a0d) --- be/src/formats/csv/binary_converter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/be/src/formats/csv/binary_converter.cpp b/be/src/formats/csv/binary_converter.cpp index 6da05dc18ea1ca..af52435b8a4471 100644 --- a/be/src/formats/csv/binary_converter.cpp +++ b/be/src/formats/csv/binary_converter.cpp @@ -3,6 +3,7 @@ #include "formats/csv/binary_converter.h" #include "column/binary_column.h" +#include "gutil/strings/substitute.h" #include "runtime/descriptors.h" #include "runtime/types.h" @@ -47,7 +48,8 @@ bool BinaryConverter::read_string(Column* column, Slice s, const Options& option } if (UNLIKELY((s.size > TypeDescriptor::MAX_VARCHAR_LENGTH) || (max_size > 0 && s.size > max_size))) { - LOG(WARNING) << "Column [" << column->get_name() << "]'s length exceed max varchar length."; + VLOG(3) << strings::Substitute("Column [$0]'s length exceed max varchar length. str_size($1), max_size($2)", + column->get_name(), s.size, max_size); return false; } down_cast(column)->append(s); @@ -92,7 +94,10 @@ bool BinaryConverter::read_quoted_string(Column* column, Slice s, const Options& size_t ext_size = new_size - old_size; if (UNLIKELY((ext_size > TypeDescriptor::MAX_VARCHAR_LENGTH) || (max_size > 0 && ext_size > max_size))) { bytes.resize(old_size); - LOG(WARNING) << "Column [" << column->get_name() << "]'s length exceed max varchar length."; + VLOG(3) << strings::Substitute( + "Column [$0]'s length exceed max varchar length. old_size($1), new_size($2), ext_size($3), " + "max_size($4)", + column->get_name(), old_size, new_size, ext_size, max_size); return false; } offsets.push_back(bytes.size()); From 39e79249d0f3f2ca28b1fb94db3ba591955e5e99 Mon Sep 17 00:00:00 2001 From: trueeyu Date: Wed, 2 Aug 2023 10:02:04 +0800 Subject: [PATCH 09/23] [Enhancement] Opt the memory usage of column zonmap for all null (#28370) Currently if the column type is varchar(length) and the values is all null, a zonemap string of length will be written to the segment file, causing the loaded metadata to occupy a large amount of memory. The main purpose of this code is to optimize the reading of segment files generated by the old version. ``` CREATE TABLE `t2` ( `c1` int(11) NULL COMMENT "", `c2` varchar(65533) NULL COMMENT "", `c3` int(11) NULL COMMENT "" ) ENGINE=OLAP DUPLICATE KEY(`c1`, `c2`) DISTRIBUTED BY HASH(`c1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", "in_memory" = "false", "enable_persistent_index" = "false", "replicated_storage" = "true", "compression" = "LZ4" ); select count(*) from t2 ; select count(*) from t2 where c3 is null; ``` 132KB -> 1KB Before optimization: ``` mysql> admin execute on 10005 'System.print(GlobalEnv.GetInstance().column_zonemap_index_mem_tracker().consumption())'; +--------+ | result | +--------+ | 132258 | +--------+ 1 row in set (0.00 sec) ``` After optimization: ``` mysql> admin execute on 10005 'System.print(GlobalEnv.GetInstance().column_zonemap_index_mem_tracker().consumption())'; +--------+ | result | +--------+ | 1056 | +--------+ 1 row in set (0.00 sec) (cherry picked from commit ac60feb6eeb5d3479a33bd63c62ec184723371a2) --- be/src/storage/rowset/zone_map_index.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/be/src/storage/rowset/zone_map_index.cpp b/be/src/storage/rowset/zone_map_index.cpp index 4b52e6b948c8a7..19c2e03749975f 100644 --- a/be/src/storage/rowset/zone_map_index.cpp +++ b/be/src/storage/rowset/zone_map_index.cpp @@ -236,6 +236,17 @@ Status ZoneMapIndexReader::_do_load(FileSystem* fs, const std::string& filename, if (!_page_zone_maps[i].ParseFromArray(value.data, value.size)) { return Status::Corruption("Failed to parse zone map"); } + + // Currently if the column type is varchar(length) and the values is all null, + // a zonemap string of length will be written to the segment file, + // causing the loaded metadata to occupy a large amount of memory. + // + // The main purpose of this code is to optimize the reading of segment files + // generated by the old version. + if (_page_zone_maps[i].has_has_not_null() && !_page_zone_maps[i].has_not_null()) { + delete _page_zone_maps[i].release_min(); + delete _page_zone_maps[i].release_max(); + } column->resize(0); } return Status::OK(); From 0d0df45a252b64858473b8819b1bb90702c9137f Mon Sep 17 00:00:00 2001 From: zhangqiang Date: Wed, 2 Aug 2023 18:09:56 +0800 Subject: [PATCH 10/23] [BugFix] Recalculate max_continuous_version after schema change finished (#28473) We will create a new tablet for each original tablet and we will write original tablet and new tablet during schema change. We will construct a `version_graph` to keep versions info for each table and the following code is how to update `version_graph`: ``` void VersionGraph::add_version_to_graph(const Version& version) { _add_version_to_graph(version); if (version.first == _max_continuous_version + 1) { _max_continuous_version = _get_max_continuous_version_from(_max_continuous_version + 1); } else if (version.first == 0) { // We need to reconstruct max_continuous_version from zero if input version is starting from zero // e.g. // 1. Tablet A is doing schema change // 2. We create a new tablet B releated A, and we will create a initial rowset and _max_continuous_version // will be updated to 1 // 3. Tablet A has a rowset R with version (0, m) // 4. Schema change will try convert R // 5. The start version of R (0) is not equal to `_max_continuous_version + 1`, and the _max_continuous_version // will not update _max_continuous_version = _get_max_continuous_version_from(0); } } ``` There exists a scenario where _max_continuous_version cannot be updated correctly e.g. 1. create a new tablet `t2` for origin tablet `t1` during schema change and `t2` has version 91,92,93,95,96,97,98. version 94 write failed. 2. when running schema change job, alter version is 98 and `t1` has version [0-90], [91-96], 97, 98 3. `max_continuous_version` of `t2` is 0 before convert rowset. After convert version [0-90], `max_continuous_version` is update to 93 because `t2` has version 91-93 before. 4. `max_continuous_version` don't update because the left rowsets version is not satisfy update condition(version.fisrt == 0 or version.first == _max_continuous_verison + 1). So the `max_continuous_version` will keep 93 after convert rowset finished. 5. we will check the max_continuous_version of `t2` at last and it should not less than alter version. But max_continuous_version(93) is less than alter version(98), so the alter job failed at last. The main reason is we don't update `_max_continuous_version` correctly during alter job, we should recalculate the `max_continuous_version` after rowset conversion. --------- Signed-off-by: zhangqiang --- be/src/storage/schema_change.cpp | 1 + be/src/storage/tablet.h | 2 ++ be/src/storage/version_graph.cpp | 8 ++++++++ be/src/storage/version_graph.h | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/be/src/storage/schema_change.cpp b/be/src/storage/schema_change.cpp index 6dbab73b4a5f5d..e45f14e476f6e3 100644 --- a/be/src/storage/schema_change.cpp +++ b/be/src/storage/schema_change.cpp @@ -1065,6 +1065,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(SchemaChangeParams& sc_p if (status.ok()) { status = sc_params.new_tablet->check_version_integrity(sc_params.version); } + sc_params.new_tablet->update_max_continuous_version(); LOG(INFO) << "finish converting rowsets for new_tablet from base_tablet. " << "base_tablet=" << sc_params.base_tablet->full_name() diff --git a/be/src/storage/tablet.h b/be/src/storage/tablet.h index b0fcde105d65a8..666eeb39850eb4 100644 --- a/be/src/storage/tablet.h +++ b/be/src/storage/tablet.h @@ -261,6 +261,8 @@ class Tablet : public BaseTablet { void get_basic_info(TabletBasicInfo& info); + void update_max_continuous_version() { _timestamped_version_tracker.update_max_continuous_version(); } + protected: void on_shutdown() override; diff --git a/be/src/storage/version_graph.cpp b/be/src/storage/version_graph.cpp index 6dc74119f9a919..f18f2b6f3100e6 100644 --- a/be/src/storage/version_graph.cpp +++ b/be/src/storage/version_graph.cpp @@ -105,6 +105,10 @@ int64_t TimestampedVersionTracker::get_max_continuous_version() const { return _version_graph.max_continuous_version(); } +void TimestampedVersionTracker::update_max_continuous_version() { + _version_graph.update_max_continuous_version(); +} + int64_t TimestampedVersionTracker::get_min_readable_version() const { return _version_graph.min_readable_version(); } @@ -223,6 +227,10 @@ std::vector& TimestampedVersionPathContainer::times return _timestamped_versions_container; } +void VersionGraph::update_max_continuous_version() { + _max_continuous_version = _get_max_continuous_version_from(0); +} + void VersionGraph::construct_version_graph(const std::vector& rs_metas, int64_t* max_version) { _version_graph.clear(); _max_continuous_version = -1; diff --git a/be/src/storage/version_graph.h b/be/src/storage/version_graph.h index d45e0bd64a8a62..291b5c278dfe7d 100644 --- a/be/src/storage/version_graph.h +++ b/be/src/storage/version_graph.h @@ -65,6 +65,8 @@ class VersionGraph { // Get max continuous version from 0 int64_t max_continuous_version() const { return _max_continuous_version; } + void update_max_continuous_version(); + int64_t min_readable_version() const { return _min_readable_version; } private: @@ -198,6 +200,8 @@ class TimestampedVersionTracker { // Get max continuous version from 0 int64_t get_max_continuous_version() const; + void update_max_continuous_version(); + int64_t get_min_readable_version() const; private: From 98f308e2f278a5782ea6984ad9e149a43fe04e41 Mon Sep 17 00:00:00 2001 From: liuzhongjun Date: Fri, 28 Apr 2023 09:48:26 +0800 Subject: [PATCH 11/23] [BugFix]Fix unrecognize database name with uppercase in udf FunctionName (#22631) --------- Signed-off-by: liuzhongjun89 --- .../com/starrocks/analysis/FunctionName.java | 5 +-- .../com/starrocks/sql/parser/AstBuilder.java | 11 +++-- .../sql/ast/CreateFunctionStmtTest.java | 41 +++++++++++++++++++ .../sql/ast/DropFunctionStmtTest.java | 35 ++++++++++++++++ 4 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 fe/fe-core/src/test/java/com/starrocks/sql/ast/CreateFunctionStmtTest.java create mode 100644 fe/fe-core/src/test/java/com/starrocks/sql/ast/DropFunctionStmtTest.java diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/FunctionName.java b/fe/fe-core/src/main/java/com/starrocks/analysis/FunctionName.java index 68e633c9ae2984..384d31bcd3827d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/FunctionName.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/FunctionName.java @@ -45,9 +45,6 @@ private FunctionName() { public FunctionName(String db, String fn) { db_ = db; fn_ = fn.toLowerCase(); - if (db_ != null) { - db_ = db_.toLowerCase(); - } } public FunctionName(String fn) { @@ -56,7 +53,7 @@ public FunctionName(String fn) { } public FunctionName(TFunctionName thriftName) { - db_ = thriftName.db_name.toLowerCase(); + db_ = thriftName.db_name; fn_ = thriftName.function_name.toLowerCase(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index edf6810c0542cf..59c08dfa90d181 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -3839,8 +3839,7 @@ public ParseNode visitShowPrivilegesStatement(StarRocksParser.ShowPrivilegesStat @Override public ParseNode visitDropFunctionStatement(StarRocksParser.DropFunctionStatementContext context) { - String functionName = getQualifiedName(context.qualifiedName()).toString().toLowerCase(); - + String functionName = getQualifiedName(context.qualifiedName()).toString(); return new DropFunctionStmt(FunctionName.createFnName(functionName), getFunctionArgsDef(context.typeList())); } @@ -3851,7 +3850,7 @@ public ParseNode visitCreateFunctionStatement(StarRocksParser.CreateFunctionStat if (context.functionType != null) { functionType = context.functionType.getText(); } - String functionName = getQualifiedName(context.qualifiedName()).toString().toLowerCase(); + String functionName = getQualifiedName(context.qualifiedName()).toString(); TypeDef returnTypeDef = new TypeDef(getType(context.returnType)); TypeDef intermediateType = null; @@ -4509,9 +4508,9 @@ private static List getArgumentsForTimeSlice(Expr time, Expr value, String @Override public ParseNode visitSimpleFunctionCall(StarRocksParser.SimpleFunctionCallContext context) { - String functionName = getQualifiedName(context.qualifiedName()).toString().toLowerCase(); - - FunctionName fnName = FunctionName.createFnName(functionName); + String fullFunctionName = getQualifiedName(context.qualifiedName()).toString(); + FunctionName fnName = FunctionName.createFnName(fullFunctionName); + String functionName = fnName.getFunction(); if (functionName.equals(FunctionSet.TIME_SLICE) || functionName.equals(FunctionSet.DATE_SLICE)) { if (context.expression().size() == 2) { Expr e1 = (Expr) visit(context.expression(0)); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/ast/CreateFunctionStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/ast/CreateFunctionStmtTest.java new file mode 100644 index 00000000000000..bef63c1052794f --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/sql/ast/CreateFunctionStmtTest.java @@ -0,0 +1,41 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.starrocks.sql.ast; + +import com.starrocks.qe.ConnectContext; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Test; + +public class CreateFunctionStmtTest { + @Mocked + private ConnectContext ctx; + + @Test + public void testNormal() throws Exception { + String createFunctionSql = "CREATE FUNCTION ABC.MY_UDF_JSON_GET(string, string) \n" + + "RETURNS string \n" + + "properties (\n" + + " \"symbol\" = \"com.starrocks.udf.sample.UDFJsonGet\",\n" + + " \"type\" = \"StarrocksJar\",\n" + + " \"file\" = \"http://http_host:http_port/udf-1.0-SNAPSHOT-jar-with-dependencies.jar\"\n" + + ");"; + CreateFunctionStmt stmt = (CreateFunctionStmt) com.starrocks.sql.parser.SqlParser.parse( + createFunctionSql, 32).get(0); + // com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); + + Assert.assertEquals("ABC", stmt.getFunctionName().getDb()); + Assert.assertEquals("my_udf_json_get", stmt.getFunctionName().getFunction()); + } +} diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/ast/DropFunctionStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/ast/DropFunctionStmtTest.java new file mode 100644 index 00000000000000..6b66e147164599 --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/sql/ast/DropFunctionStmtTest.java @@ -0,0 +1,35 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.starrocks.sql.ast; + +import com.starrocks.qe.ConnectContext; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Test; + +public class DropFunctionStmtTest { + @Mocked + private ConnectContext ctx; + + @Test + public void testNormal() throws Exception { + String dropFunctionSql = "DROP FUNCTION ABC.MY_UDF_JSON_GET(string, string)"; + DropFunctionStmt stmt = (DropFunctionStmt) com.starrocks.sql.parser.SqlParser.parse( + dropFunctionSql, 32).get(0); + // com.starrocks.sql.analyzer.Analyzer.analyze(stmt, ctx); + + Assert.assertEquals("ABC", stmt.getFunctionName().getDb()); + Assert.assertEquals("my_udf_json_get", stmt.getFunctionName().getFunction()); + } +} From 1a3aad6b3c673761d59a00c4cd2da7736b27f810 Mon Sep 17 00:00:00 2001 From: luohaha <18810541851@163.com> Date: Tue, 1 Aug 2023 17:53:05 +0800 Subject: [PATCH 12/23] [Enhancement] improve user prompt about too many versions error Signed-off-by: luohaha <18810541851@163.com> --- be/src/storage/delta_writer.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/be/src/storage/delta_writer.cpp b/be/src/storage/delta_writer.cpp index ea7463e3288a11..1005df3ab41e27 100644 --- a/be/src/storage/delta_writer.cpp +++ b/be/src/storage/delta_writer.cpp @@ -121,8 +121,12 @@ Status DeltaWriter::_init() { if (config::enable_event_based_compaction_framework) { StorageEngine::instance()->compaction_manager()->update_tablet_async(_tablet); } - auto msg = fmt::format("Too many versions. tablet_id: {}, version_count: {}, limit: {}, replica_state: {}", - _opt.tablet_id, _tablet->version_count(), config::tablet_max_versions, _replica_state); + auto msg = fmt::format( + "Failed to load data into tablet {}, because of too many versions, current/limit: {}/{}. You can " + "reduce the loading job concurrency, or increase loading data batch size. If you are loading data with " + "Routine Load, you can increase FE configs routine_load_task_consume_second and " + "max_routine_load_batch_size,", + _opt.tablet_id, _tablet->version_count(), config::tablet_max_versions); LOG(ERROR) << msg; Status st = Status::ServiceUnavailable(msg); _set_state(kUninitialized, st); From f26b9ee7e1d81b0c52af08cec307148fc4e49d3d Mon Sep 17 00:00:00 2001 From: luohaha <18810541851@163.com> Date: Tue, 1 Aug 2023 20:48:06 +0800 Subject: [PATCH 13/23] [Enhancement] improve usert prompt when data loading meets drop table or schema change Signed-off-by: luohaha <18810541851@163.com> --- be/src/storage/delta_writer.cpp | 4 +++- be/src/storage/tablet_updates.cpp | 25 ++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/be/src/storage/delta_writer.cpp b/be/src/storage/delta_writer.cpp index 1005df3ab41e27..e60c7bf859127a 100644 --- a/be/src/storage/delta_writer.cpp +++ b/be/src/storage/delta_writer.cpp @@ -90,7 +90,9 @@ Status DeltaWriter::_init() { _tablet = tablet_mgr->get_tablet(_opt.tablet_id, false); if (_tablet == nullptr) { std::stringstream ss; - ss << "Fail to get tablet. tablet_id=" << _opt.tablet_id; + ss << "Fail to get tablet, perhaps this table is doing schema change, or it has already been deleted. Please " + "try again. tablet_id=" + << _opt.tablet_id; LOG(WARNING) << ss.str(); Status st = Status::InternalError(ss.str()); _set_state(kUninitialized, st); diff --git a/be/src/storage/tablet_updates.cpp b/be/src/storage/tablet_updates.cpp index cd4ce194a8dcb7..b32d10f998de0e 100644 --- a/be/src/storage/tablet_updates.cpp +++ b/be/src/storage/tablet_updates.cpp @@ -508,7 +508,10 @@ Status TabletUpdates::_get_apply_version_and_rowsets(int64_t* version, std::vect std::vector* rowset_ids) { std::lock_guard rl(_lock); if (_edit_version_infos.empty()) { - string msg = Substitute("tablet deleted when _get_apply_version_and_rowsets tablet:$0", _tablet.tablet_id()); + string msg = strings::Substitute( + "Tablet is deleted, perhaps this table is doing schema change, or it has already been deleted. Please " + "try again. get_apply_version_and_rowsets tablet:$0", + _tablet.tablet_id()); LOG(WARNING) << msg; return Status::InternalError(msg); } @@ -543,7 +546,10 @@ Status TabletUpdates::rowset_commit(int64_t version, const RowsetSharedPtr& rows { std::unique_lock ul(_lock); if (_edit_version_infos.empty()) { - string msg = Substitute("tablet deleted when rowset_commit tablet:$0", _tablet.tablet_id()); + string msg = strings::Substitute( + "Tablet is deleted, perhaps this table is doing schema change, or it has already been deleted. " + "Please try again. rowset_commit tablet:$0", + _tablet.tablet_id()); LOG(WARNING) << msg; return Status::InternalError(msg); } @@ -830,7 +836,10 @@ void TabletUpdates::_stop_and_wait_apply_done() { Status TabletUpdates::get_latest_applied_version(EditVersion* latest_applied_version) { std::lock_guard l(_lock); if (_edit_version_infos.empty()) { - string msg = Substitute("tablet deleted when get_latest_applied_version tablet:$0", _tablet.tablet_id()); + string msg = strings::Substitute( + "Tablet is deleted, perhaps this table is doing schema change, or it has already been deleted. " + "get_latest_applied_version tablet:$0", + _tablet.tablet_id()); LOG(WARNING) << msg; return Status::InternalError(msg); } @@ -1242,7 +1251,10 @@ RowsetSharedPtr TabletUpdates::_get_rowset(uint32_t rowset_id) { Status TabletUpdates::_wait_for_version(const EditVersion& version, int64_t timeout_ms, std::unique_lock& ul) { if (_edit_version_infos.empty()) { - string msg = Substitute("tablet deleted when _wait_for_version tablet:$0", _tablet.tablet_id()); + string msg = strings::Substitute( + "Tablet is deleted, perhaps this table is doing schema change, or it has already been deleted. " + "_wait_for_version tablet:$0", + _tablet.tablet_id()); LOG(WARNING) << msg; return Status::InternalError(msg); } @@ -2565,7 +2577,10 @@ Status TabletUpdates::get_applied_rowsets(int64_t version, std::vector Date: Wed, 26 Jul 2023 08:42:00 +0800 Subject: [PATCH 14/23] [Feature] support create or replace view (#27768) Fixes #27767 `CREATE OR REPLACE VIEW` could be used to update existed view definition. (cherry picked from commit 6e1d5ec99bc54ca0b03a3c1cf7ac2253e24512ee) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/alter/Alter.java # fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java # fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java # fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java # fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java --- .../main/java/com/starrocks/alter/Alter.java | 5 +++ .../com/starrocks/server/LocalMetastore.java | 42 ++++++++++++++++++- .../com/starrocks/sql/ast/AlterViewStmt.java | 12 ++++++ .../com/starrocks/sql/ast/CreateViewStmt.java | 22 ++++++++++ .../com/starrocks/sql/parser/AstBuilder.java | 5 +++ .../com/starrocks/sql/parser/StarRocks.g4 | 2 +- .../com/starrocks/catalog/CreateViewTest.java | 42 ++++++++++++++++++- .../starrocks/utframe/StarRocksAssert.java | 6 ++- 8 files changed, 132 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java b/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java index 8a206eeacb20ee..d193d75b113eec 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java @@ -871,8 +871,13 @@ private void swapTableInternal(Database db, OlapTable origTable, OlapTable newTb db.createTable(origTable); } +<<<<<<< HEAD:fe/fe-core/src/main/java/com/starrocks/alter/Alter.java public void processAlterView(AlterViewStmt stmt, ConnectContext ctx) throws UserException { TableName dbTableName = stmt.getTbl(); +======= + public void processAlterView(AlterViewStmt stmt, ConnectContext ctx) throws DdlException { + TableName dbTableName = stmt.getTableName(); +>>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)):fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java String dbName = dbTableName.getDb(); Database db = GlobalStateMgr.getCurrentState().getDb(dbName); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 278f26cf4e6c0e..1caedb2b4708ae 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -31,8 +31,15 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +<<<<<<< HEAD import com.staros.proto.ShardStorageInfo; import com.starrocks.analysis.ColumnDef; +======= +import com.staros.proto.FilePathInfo; +import com.starrocks.alter.AlterJobMgr; +import com.starrocks.analysis.Expr; +import com.starrocks.analysis.FunctionCallExpr; +>>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) import com.starrocks.analysis.IntLiteral; import com.starrocks.analysis.KeysDesc; import com.starrocks.analysis.StringLiteral; @@ -4240,12 +4247,16 @@ public void createView(CreateViewStmt stmt) throws DdlException { } // check if table exists in db + boolean existed = false; db.readLock(); try { if (db.getTable(tableName) != null) { + existed = true; if (stmt.isSetIfNotExists()) { LOG.info("create view[{}] which already exists", tableName); return; + } else if (stmt.isReplace()) { + LOG.info("view {} already exists, need to replace it", tableName); } else { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } @@ -4254,8 +4265,32 @@ public void createView(CreateViewStmt stmt) throws DdlException { db.readUnlock(); } - List columns = stmt.getColumns(); + if (existed) { + // already existed, need to alter the view + AlterJobMgr alterJobMgr = GlobalStateMgr.getCurrentState().getAlterJobMgr(); + try { + AlterViewStmt alterViewStmt = AlterViewStmt.fromReplaceStmt(stmt); + alterJobMgr.processAlterView(alterViewStmt, ConnectContext.get()); + LOG.info("replace view {} successfully", tableName); + } catch (DdlException e) { + LOG.warn("replace view failed due to {}", e.getMessage(), e); + throw new DdlException("replace view failed due to " + e.getMessage(), e); + } + } else { + List columns = stmt.getColumns(); + long tableId = getNextId(); + View view = new View(tableId, tableName, columns); + view.setComment(stmt.getComment()); + view.setInlineViewDefWithSqlMode(stmt.getInlineViewDef(), + ConnectContext.get().getSessionVariable().getSqlMode()); + // init here in case the stmt string from view.toSql() has some syntax error. + try { + view.init(); + } catch (UserException e) { + throw new DdlException("failed to init view stmt", e); + } +<<<<<<< HEAD long tableId = getNextId(); View newView = new View(tableId, tableName, columns); newView.setComment(stmt.getComment()); @@ -4289,6 +4324,11 @@ public void createView(CreateViewStmt stmt) throws DdlException { } LOG.info("successfully create view[" + tableName + "-" + newView.getId() + "]"); +======= + onCreate(db, view, "", stmt.isSetIfNotExists()); + LOG.info("successfully create view[" + tableName + "-" + view.getId() + "]"); + } +>>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) } public void replayCreateCluster(Cluster cluster) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java index a7fc90c41c641b..bdefbadcbd70fe 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java @@ -12,7 +12,19 @@ public AlterViewStmt(TableName tbl, List cols, QueryStatement qu super(tbl, cols, queryStatement); } +<<<<<<< HEAD public TableName getTbl() { +======= + public static AlterViewStmt fromReplaceStmt(CreateViewStmt stmt) { + AlterViewClause alterViewClause = new AlterViewClause( + stmt.getColWithComments(), stmt.getQueryStatement(), NodePosition.ZERO); + alterViewClause.setInlineViewDef(stmt.getInlineViewDef()); + alterViewClause.setColumns(stmt.getColumns()); + return new AlterViewStmt(stmt.getTableName(), alterViewClause, NodePosition.ZERO); + } + + public TableName getTableName() { +>>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) return tableName; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java index 5842baeecc4fa6..033fa4b74d133a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java @@ -9,12 +9,30 @@ public class CreateViewStmt extends BaseViewStmt { private final boolean ifNotExists; + private final boolean replace; private final String comment; +<<<<<<< HEAD public CreateViewStmt(boolean ifNotExists, TableName tableName, List cols, String comment, QueryStatement queryStmt) { super(tableName, cols, queryStmt); this.ifNotExists = ifNotExists; +======= + //Resolved by Analyzer + protected List columns; + private String inlineViewDef; + + public CreateViewStmt(boolean ifNotExists, boolean replace, + TableName tableName, List colWithComments, + String comment, + QueryStatement queryStmt, + NodePosition pos) { + super(pos); + this.ifNotExists = ifNotExists; + this.replace = replace; + this.tableName = tableName; + this.colWithComments = colWithComments; +>>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) this.comment = Strings.nullToEmpty(comment); } @@ -22,6 +40,10 @@ public boolean isSetIfNotExists() { return ifNotExists; } + public boolean isReplace() { + return replace; + } + public String getComment() { return comment; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 59c08dfa90d181..102bfa0aa33cdf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -918,8 +918,13 @@ public ParseNode visitCreateViewStatement(StarRocksParser.CreateViewStatementCon if (context.columnNameWithComment().size() > 0) { colWithComments = visit(context.columnNameWithComment(), ColWithComment.class); } + if (context.IF() != null && context.REPLACE() != null) { + throw new ParsingException(PARSER_ERROR_MSG.conflictedOptions("if not exists", "or replace"), + createPos(context)); + } return new CreateViewStmt( context.IF() != null, + context.REPLACE() != null, targetTableName, colWithComments, context.comment() == null ? null : ((StringLiteral) visit(context.comment())).getStringValue(), diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index f8e7ac768e76e1..49c67a917e5478 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -467,7 +467,7 @@ recoverPartitionStatement // ------------------------------------------- View Statement ---------------------------------------------------------- createViewStatement - : CREATE VIEW (IF NOT EXISTS)? qualifiedName + : CREATE (OR REPLACE)? VIEW (IF NOT EXISTS)? qualifiedName ('(' columnNameWithComment (',' columnNameWithComment)* ')')? comment? AS queryStatement ; diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java index f8122c68b0be06..36d1b283b6fc97 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java @@ -89,4 +89,44 @@ public void testCreateViewNullable() throws Exception { Assert.assertTrue(column.isAllowNull()); } } -} \ No newline at end of file +<<<<<<< HEAD +} +======= + + @Test + public void createReplace() throws Exception { + StarRocksAssert starRocksAssert = new StarRocksAssert(connectContext); + starRocksAssert.useDatabase("test"); + starRocksAssert.withTable("CREATE TABLE `test_replace_site_access` (\n" + + " `event_day` date NULL COMMENT \"\",\n" + + " `site_id` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" + + " `city_code` varchar(100) NULL COMMENT \"\",\n" + + " `user_name` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" + + " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" + + ") ENGINE=OLAP \n" + + "DUPLICATE KEY(`event_day`, `site_id`, `city_code`, `user_name`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`event_day`, `site_id`) BUCKETS 32 \n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"enable_persistent_index\" = \"false\",\n" + + "\"compression\" = \"LZ4\"\n" + + ");"); + + // create non existed view + starRocksAssert.withView("create or replace view test_null_view as select event_day " + + "from test_replace_site_access;"); + Assert.assertNotNull(starRocksAssert.getTable("test", "test_null_view")); + + // replace existed view + starRocksAssert.withView("create or replace view test_null_view as select site_id " + + "from test_replace_site_access;"); + View view = (View) starRocksAssert.getTable("test", "test_null_view"); + Assert.assertEquals( + "SELECT `test`.`test_replace_site_access`.`site_id`\nFROM `test`.`test_replace_site_access`", + view.getInlineViewDef()); + Assert.assertNotNull(view.getColumn("site_id")); + } +} +>>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java b/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java index 696dcf79df8f0c..e56dbde3ba46d5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/StarRocksAssert.java @@ -165,9 +165,13 @@ public StarRocksAssert withTable(String sql) throws Exception { return this; } + public Table getTable(String dbName, String tableName) { + return ctx.getGlobalStateMgr().mayGetDb(dbName).map(db -> db.getTable(tableName)).orElse(null); + } + public StarRocksAssert withSingleReplicaTable(String sql) throws Exception { StatementBase statementBase = UtFrameUtils.parseStmtWithNewParser(sql, ctx); - if (statementBase instanceof CreateTableStmt) { + if (statementBase instanceof CreateTableStmt) { CreateTableStmt createTableStmt = (CreateTableStmt) statementBase; createTableStmt.getProperties().put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "1"); return this.withTable(sql); From 4f7c65846910c0b56e43dd70ad0371d084570d03 Mon Sep 17 00:00:00 2001 From: Murphy Date: Wed, 26 Jul 2023 15:40:11 +0800 Subject: [PATCH 15/23] fix conflict Signed-off-by: Murphy --- .../main/java/com/starrocks/alter/Alter.java | 5 -- .../com/starrocks/server/GlobalStateMgr.java | 8 +-- .../com/starrocks/server/LocalMetastore.java | 49 +------------------ .../com/starrocks/sql/ast/AlterViewStmt.java | 14 ++---- .../com/starrocks/sql/ast/CreateViewStmt.java | 20 +------- .../com/starrocks/sql/parser/AstBuilder.java | 3 +- .../com/starrocks/catalog/CreateViewTest.java | 4 -- 7 files changed, 12 insertions(+), 91 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java b/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java index d193d75b113eec..43e29de4e3db4c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/Alter.java @@ -871,13 +871,8 @@ private void swapTableInternal(Database db, OlapTable origTable, OlapTable newTb db.createTable(origTable); } -<<<<<<< HEAD:fe/fe-core/src/main/java/com/starrocks/alter/Alter.java - public void processAlterView(AlterViewStmt stmt, ConnectContext ctx) throws UserException { - TableName dbTableName = stmt.getTbl(); -======= public void processAlterView(AlterViewStmt stmt, ConnectContext ctx) throws DdlException { TableName dbTableName = stmt.getTableName(); ->>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)):fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java String dbName = dbTableName.getDb(); Database db = GlobalStateMgr.getCurrentState().getDb(dbName); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index 666b951d47fe67..aac32bdca09198 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -2643,6 +2643,10 @@ public String getToken() { return nodeMgr.getToken(); } + public Optional mayGetDb(String name) { + return Optional.ofNullable(localMetastore.getDb(name)); + } + public Database getDb(String name) { return localMetastore.getDb(name); } @@ -2651,10 +2655,6 @@ public Optional
mayGetTable(long dbId, long tableId) { return mayGetDb(dbId).flatMap(db -> db.tryGetTable(tableId)); } - public Optional mayGetDb(String name) { - return Optional.ofNullable(localMetastore.getDb(name)); - } - public Optional mayGetDb(long dbId) { return Optional.ofNullable(localMetastore.getDb(dbId)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 1caedb2b4708ae..d2bcfae6c7ffa1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -31,15 +31,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -<<<<<<< HEAD import com.staros.proto.ShardStorageInfo; import com.starrocks.analysis.ColumnDef; -======= -import com.staros.proto.FilePathInfo; -import com.starrocks.alter.AlterJobMgr; -import com.starrocks.analysis.Expr; -import com.starrocks.analysis.FunctionCallExpr; ->>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) import com.starrocks.analysis.IntLiteral; import com.starrocks.analysis.KeysDesc; import com.starrocks.analysis.StringLiteral; @@ -4267,12 +4260,11 @@ public void createView(CreateViewStmt stmt) throws DdlException { if (existed) { // already existed, need to alter the view - AlterJobMgr alterJobMgr = GlobalStateMgr.getCurrentState().getAlterJobMgr(); try { AlterViewStmt alterViewStmt = AlterViewStmt.fromReplaceStmt(stmt); - alterJobMgr.processAlterView(alterViewStmt, ConnectContext.get()); + alterView(alterViewStmt); LOG.info("replace view {} successfully", tableName); - } catch (DdlException e) { + } catch (UserException e) { LOG.warn("replace view failed due to {}", e.getMessage(), e); throw new DdlException("replace view failed due to " + e.getMessage(), e); } @@ -4290,45 +4282,8 @@ public void createView(CreateViewStmt stmt) throws DdlException { throw new DdlException("failed to init view stmt", e); } -<<<<<<< HEAD - long tableId = getNextId(); - View newView = new View(tableId, tableName, columns); - newView.setComment(stmt.getComment()); - newView.setInlineViewDefWithSqlMode(stmt.getInlineViewDef(), - ConnectContext.get().getSessionVariable().getSqlMode()); - // init here in case the stmt string from view.toSql() has some syntax error. - try { - newView.init(); - } catch (UserException e) { - throw new DdlException("failed to init view stmt", e); - } - - // check database exists again, because database can be dropped when creating table - if (!tryLock(false)) { - throw new DdlException("Failed to acquire globalStateMgr lock. Try again"); - } - try { - if (getDb(db.getId()) == null) { - throw new DdlException("database has been dropped when creating view"); - } - if (!db.createTableWithLock(newView, false)) { - if (!stmt.isSetIfNotExists()) { - ErrorReport.reportDdlException(ErrorCode.ERR_CANT_CREATE_TABLE, tableName, "table already exists"); - } else { - LOG.info("create table[{}] which already exists", tableName); - return; - } - } - } finally { - unlock(); - } - - LOG.info("successfully create view[" + tableName + "-" + newView.getId() + "]"); -======= - onCreate(db, view, "", stmt.isSetIfNotExists()); LOG.info("successfully create view[" + tableName + "-" + view.getId() + "]"); } ->>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) } public void replayCreateCluster(Cluster cluster) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java index bdefbadcbd70fe..3eea7f1b5a9f89 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java @@ -12,20 +12,12 @@ public AlterViewStmt(TableName tbl, List cols, QueryStatement qu super(tbl, cols, queryStatement); } -<<<<<<< HEAD public TableName getTbl() { -======= - public static AlterViewStmt fromReplaceStmt(CreateViewStmt stmt) { - AlterViewClause alterViewClause = new AlterViewClause( - stmt.getColWithComments(), stmt.getQueryStatement(), NodePosition.ZERO); - alterViewClause.setInlineViewDef(stmt.getInlineViewDef()); - alterViewClause.setColumns(stmt.getColumns()); - return new AlterViewStmt(stmt.getTableName(), alterViewClause, NodePosition.ZERO); + return tableName; } - public TableName getTableName() { ->>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) - return tableName; + public static AlterViewStmt fromReplaceStmt(CreateViewStmt stmt) { + return new AlterViewStmt(stmt.getTableName(), stmt.getCols(), stmt.getQueryStatement()); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java index 033fa4b74d133a..6581fab4cd28eb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/CreateViewStmt.java @@ -12,27 +12,11 @@ public class CreateViewStmt extends BaseViewStmt { private final boolean replace; private final String comment; -<<<<<<< HEAD - public CreateViewStmt(boolean ifNotExists, TableName tableName, List cols, + public CreateViewStmt(boolean ifNotExists, boolean replace, TableName tableName, List cols, String comment, QueryStatement queryStmt) { super(tableName, cols, queryStmt); - this.ifNotExists = ifNotExists; -======= - //Resolved by Analyzer - protected List columns; - private String inlineViewDef; - - public CreateViewStmt(boolean ifNotExists, boolean replace, - TableName tableName, List colWithComments, - String comment, - QueryStatement queryStmt, - NodePosition pos) { - super(pos); - this.ifNotExists = ifNotExists; this.replace = replace; - this.tableName = tableName; - this.colWithComments = colWithComments; ->>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) + this.ifNotExists = ifNotExists; this.comment = Strings.nullToEmpty(comment); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 102bfa0aa33cdf..5d59e841441ba4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -919,8 +919,7 @@ public ParseNode visitCreateViewStatement(StarRocksParser.CreateViewStatementCon colWithComments = visit(context.columnNameWithComment(), ColWithComment.class); } if (context.IF() != null && context.REPLACE() != null) { - throw new ParsingException(PARSER_ERROR_MSG.conflictedOptions("if not exists", "or replace"), - createPos(context)); + throw new ParsingException("conflicted option IF NOT EXISTS and OR REPLACE"); } return new CreateViewStmt( context.IF() != null, diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java index 36d1b283b6fc97..172d9e31764d2d 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CreateViewTest.java @@ -89,9 +89,6 @@ public void testCreateViewNullable() throws Exception { Assert.assertTrue(column.isAllowNull()); } } -<<<<<<< HEAD -} -======= @Test public void createReplace() throws Exception { @@ -129,4 +126,3 @@ public void createReplace() throws Exception { Assert.assertNotNull(view.getColumn("site_id")); } } ->>>>>>> 6e1d5ec99b ([Feature] support create or replace view (#27768)) From 2d3f4570e8dd208627fc0cc6e3405564e967c91c Mon Sep 17 00:00:00 2001 From: Murphy Date: Wed, 2 Aug 2023 13:53:40 +0800 Subject: [PATCH 16/23] fix create view Signed-off-by: Murphy --- .../com/starrocks/server/LocalMetastore.java | 21 +++++++++++++++++++ .../com/starrocks/sql/ast/AlterViewStmt.java | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index d2bcfae6c7ffa1..a53bf5d2a70feb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -4282,6 +4282,27 @@ public void createView(CreateViewStmt stmt) throws DdlException { throw new DdlException("failed to init view stmt", e); } + // check database exists again, because database can be dropped when creating table + if (!tryLock(false)) { + throw new DdlException("Failed to acquire globalStateMgr lock. Try again"); + } + try { + if (getDb(db.getId()) == null) { + throw new DdlException("database has been dropped when creating view"); + } + if (!db.createTableWithLock(view, false)) { + if (!stmt.isSetIfNotExists()) { + ErrorReport.reportDdlException(ErrorCode.ERR_CANT_CREATE_TABLE, tableName, + "table already exists"); + } else { + LOG.info("create table[{}] which already exists", tableName); + return; + } + } + } finally { + unlock(); + } + LOG.info("successfully create view[" + tableName + "-" + view.getId() + "]"); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java index 3eea7f1b5a9f89..479178ef756448 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterViewStmt.java @@ -17,7 +17,10 @@ public TableName getTbl() { } public static AlterViewStmt fromReplaceStmt(CreateViewStmt stmt) { - return new AlterViewStmt(stmt.getTableName(), stmt.getCols(), stmt.getQueryStatement()); + AlterViewStmt res = new AlterViewStmt(stmt.getTableName(), stmt.getCols(), stmt.getQueryStatement()); + res.setInlineViewDef(stmt.getInlineViewDef()); + res.setFinalCols(stmt.getColumns()); + return res; } @Override From 316d411c8e2fa8123b368d2a423d729d7e5ab5de Mon Sep 17 00:00:00 2001 From: xiangguangyxg <110401425+xiangguangyxg@users.noreply.github.com> Date: Thu, 3 Aug 2023 15:14:45 +0800 Subject: [PATCH 17/23] [Enhancement] External olap table support temp partition (#28389) (#28480) Signed-off-by: xiangguangyxg --- .../starrocks/catalog/ExternalOlapTable.java | 9 +- .../com/starrocks/catalog/PartitionType.java | 1 + .../java/com/starrocks/leader/LeaderImpl.java | 9 +- .../starrocks/TableMetaSyncerTest.java | 139 ++++++++++++++++++ gensrc/thrift/FrontendService.thrift | 2 + 5 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java index dcc8da228f76a4..c213c8a025b57e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ExternalOlapTable.java @@ -372,7 +372,8 @@ public void updateMeta(String dbName, TTableMeta meta, List backen thriftDataProperty.getCold_time()); // TODO: confirm false is ok RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - rangePartitionInfo.addPartition(partitionId, false, range, dataProperty, replicaNum, inMemory); + rangePartitionInfo.addPartition(partitionId, tRange.isSetIs_temp() && tRange.isIs_temp(), + range, dataProperty, replicaNum, inMemory); } break; case UNPARTITIONED: @@ -488,7 +489,11 @@ public void updateMeta(String dbName, TTableMeta meta, List backen } } } - addPartition(partition); + if (partitionMeta.isSetIs_temp() && partitionMeta.isIs_temp()) { + addTempPartition(partition); + } else { + addPartition(partition); + } } long endOfTabletMetaBuild = System.currentTimeMillis(); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionType.java b/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionType.java index 118fadc4b99fe6..50926b2db34436 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionType.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/PartitionType.java @@ -51,6 +51,7 @@ public TPartitionType toThrift() { case UNPARTITIONED: return TPartitionType.UNPARTITIONED; case RANGE: + case EXPR_RANGE: return TPartitionType.RANGE_PARTITIONED; default: return TPartitionType.UNPARTITIONED; diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java index e53d494b6c8a81..a66024a4734768 100644 --- a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java +++ b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.starrocks.alter.AlterJobV2.JobType; import com.starrocks.catalog.Column; @@ -886,6 +887,7 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { partitionMeta.setVisible_version(partition.getVisibleVersion()); partitionMeta.setVisible_time(partition.getVisibleVersionTime()); partitionMeta.setNext_version(partition.getNextVersion()); + partitionMeta.setIs_temp(olapTable.getPartition(partition.getName(), true) != null); tableMeta.addToPartitions(partitionMeta); Short replicaNum = partitionInfo.getReplicationNum(partition.getId()); boolean inMemory = partitionInfo.getIsInMemory(partition.getId()); @@ -914,7 +916,9 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { columnMeta.setComment(column.getComment()); rangePartitionDesc.addToColumns(columnMeta); } - Map> ranges = rangePartitionInfo.getIdToRange(false); + Map> ranges = Maps.newHashMap(rangePartitionInfo.getIdToRange(false)); + Map> tempRanges = rangePartitionInfo.getIdToRange(true); + ranges.putAll(tempRanges); for (Map.Entry> range : ranges.entrySet()) { TRange tRange = new TRange(); tRange.setPartition_id(range.getKey()); @@ -928,6 +932,7 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { range.getValue().upperEndpoint().write(stream); tRange.setEnd_key(output.toByteArray()); tRange.setBase_desc(basePartitionDesc); + tRange.setIs_temp(tempRanges.containsKey(range.getKey())); rangePartitionDesc.putToRanges(range.getKey(), tRange); } tPartitionInfo.setRange_partition_desc(rangePartitionDesc); @@ -954,7 +959,7 @@ public TGetTableMetaResponse getTableMeta(TGetTableMetaRequest request) { tableMeta.addToIndex_infos(indexInfo); } - for (Partition partition : olapTable.getPartitions()) { + for (Partition partition : olapTable.getAllPartitions()) { List indexes = partition.getMaterializedIndices(IndexExtState.ALL); for (MaterializedIndex index : indexes) { TIndexMeta indexMeta = new TIndexMeta(); diff --git a/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java b/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java new file mode 100644 index 00000000000000..1569e6d126906e --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/external/starrocks/TableMetaSyncerTest.java @@ -0,0 +1,139 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package com.starrocks.external.starrocks; + +import com.starrocks.catalog.ExternalOlapTable; +import com.starrocks.catalog.Table; +import com.starrocks.leader.LeaderImpl; +import com.starrocks.qe.DDLStmtExecutor; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.analyzer.AnalyzeTestUtil; +import com.starrocks.thrift.TGetTableMetaRequest; +import com.starrocks.thrift.TGetTableMetaResponse; +import com.starrocks.utframe.StarRocksAssert; +import com.starrocks.utframe.UtFrameUtils; +import org.junit.BeforeClass; +import org.junit.Test; + +import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeSuccess; + +public class TableMetaSyncerTest { + private static StarRocksAssert starRocksAssert; + + @BeforeClass + public static void beforeClass() throws Exception { + UtFrameUtils.createMinStarRocksCluster(); + AnalyzeTestUtil.init(); + starRocksAssert = new StarRocksAssert(AnalyzeTestUtil.getConnectContext()); + starRocksAssert.withDatabase("test_db").useDatabase("test_db"); + } + + @Test + public void syncTableMeta() throws Exception { + DDLStmtExecutor.execute(analyzeSuccess( + "CREATE TABLE test_table" + + "(" + + "event_time DATETIME," + + "channel VARCHAR(32) DEFAULT ''," + + "user VARCHAR(128) DEFAULT ''," + + "is_anonymous TINYINT DEFAULT '0'," + + "is_minor TINYINT DEFAULT '0'," + + "is_new TINYINT DEFAULT '0'," + + "is_robot TINYINT DEFAULT '0'," + + "is_unpatrolled TINYINT DEFAULT '0'," + + "delta INT DEFAULT '0'," + + "added INT DEFAULT '0'," + + "deleted INT DEFAULT '0'" + + ")" + + "DUPLICATE KEY" + + "(" + + "event_time," + + "channel," + + "user," + + "is_anonymous," + + "is_minor," + + "is_new," + + "is_robot," + + "is_unpatrolled" + + ")" + + "PARTITION BY RANGE(event_time)" + + "(" + + "PARTITION p06 VALUES LESS THAN ('2015-09-12 06:00:00')," + + "PARTITION p12 VALUES LESS THAN ('2015-09-12 12:00:00')," + + "PARTITION p18 VALUES LESS THAN ('2015-09-12 18:00:00')," + + "PARTITION p24 VALUES LESS THAN ('2015-09-13 00:00:00')" + + ")" + + "DISTRIBUTED BY HASH(user)" + + "properties (" + + "\"replication_num\" = \"1\"" + + ")"), starRocksAssert.getCtx()); + + + DDLStmtExecutor.execute(analyzeSuccess( + "CREATE EXTERNAL TABLE test_ext_table" + + "(" + + "event_time DATETIME," + + "channel VARCHAR(32) DEFAULT ''," + + "user VARCHAR(128) DEFAULT ''," + + "is_anonymous TINYINT DEFAULT '0'," + + "is_minor TINYINT DEFAULT '0'," + + "is_new TINYINT DEFAULT '0'," + + "is_robot TINYINT DEFAULT '0'," + + "is_unpatrolled TINYINT DEFAULT '0'," + + "delta INT DEFAULT '0'," + + "added INT DEFAULT '0'," + + "deleted INT DEFAULT '0'" + + ")" + + "DUPLICATE KEY" + + "(" + + "event_time," + + "channel," + + "user," + + "is_anonymous," + + "is_minor," + + "is_new," + + "is_robot," + + "is_unpatrolled" + + ")" + + "PARTITION BY RANGE(event_time)" + + "(" + + "PARTITION p06 VALUES LESS THAN ('2015-09-12 06:00:00')," + + "PARTITION p12 VALUES LESS THAN ('2015-09-12 12:00:00')," + + "PARTITION p18 VALUES LESS THAN ('2015-09-12 18:00:00')," + + "PARTITION p24 VALUES LESS THAN ('2015-09-13 00:00:00')" + + ")" + + "DISTRIBUTED BY HASH(user)" + + "properties (" + + "\"host\" = \"127.0.0.2\"," + + "\"port\" = \"9020\"," + + "\"user\" = \"root\"," + + "\"password\" = \"\"," + + "\"database\" = \"test_db\"," + + "\"table\" = \"test_ext_table\"" + + ")"), starRocksAssert.getCtx()); + + TGetTableMetaRequest request = new TGetTableMetaRequest(); + request.setDb_name("test_db"); + request.setTable_name("test_table"); + + LeaderImpl leader = new LeaderImpl(); + TGetTableMetaResponse response = leader.getTableMeta(request); + + Table table = GlobalStateMgr.getCurrentState().getDb("test_db").getTable("test_ext_table"); + ExternalOlapTable extTable = (ExternalOlapTable) table; + extTable.updateMeta(request.getDb_name(), response.getTable_meta(), response.getBackends()); + } +} diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 060a369c20517d..ee2fa1df3b3309 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -878,6 +878,7 @@ struct TRange { 2: optional TBasePartitionDesc base_desc 3: optional binary start_key 4: optional binary end_key + 5: optional bool is_temp } struct TRangePartitionDesc { @@ -903,6 +904,7 @@ struct TPartitionMeta { 7: optional i64 visible_time 8: optional i64 next_version 9: optional i64 next_version_hash // Deprecated + 10: optional bool is_temp } struct THashDistributionInfo { From 9eea73e022563219f8a5774e068da9ddc862522e Mon Sep 17 00:00:00 2001 From: Astralidea Date: Thu, 3 Aug 2023 14:46:47 +0800 Subject: [PATCH 18/23] Tips for optimizing replication_num when creating a table Signed-off-by: Astralidea --- .../common/util/PropertyAnalyzer.java | 8 +++--- .../com/starrocks/server/LocalMetastore.java | 26 ++++++++++++++----- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index a1dfdf27fa105c..8382387bd4db00 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -22,6 +22,7 @@ package com.starrocks.common.util; import com.clearspring.analytics.util.Lists; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.base.Strings; @@ -366,9 +367,10 @@ private static void checkAvailableBackendsIsEnough(short replicationNum) throws } List backendIds = GlobalStateMgr.getCurrentSystemInfo().getAvailableBackendIds(); if (replicationNum > backendIds.size()) { - throw new AnalysisException("Replication num should be less than the number of available BE nodes. " - + "Replication num is " + replicationNum + " available BE nodes is " + backendIds.size() + - ", You can change this default by setting the replication_num table properties."); + throw new AnalysisException("Table replication num should be less than " + + "of equal to the number of available BE nodes. " + + "You can change this default by setting the replication_num table properties. " + + "Current alive backend is [" + Joiner.on(",").join(backendIds) + "]."); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index a53bf5d2a70feb..2df88981e22600 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -1601,7 +1601,7 @@ private Partition createPartitionCommon(Database db, OlapTable table, long parti createLakeTablets((LakeTable) table, partitionId, index, distributionInfo, replicationNum, tabletMeta, tabletIdSet); } else { - createOlapTablets(index, Replica.ReplicaState.NORMAL, distributionInfo, + createOlapTablets(table, index, Replica.ReplicaState.NORMAL, distributionInfo, partition.getVisibleVersion(), replicationNum, tabletMeta, tabletIdSet); } if (index.getId() != table.getBaseIndexId()) { @@ -2086,15 +2086,20 @@ private void createOlapOrLakeTable(Database db, CreateTableStmt stmt) throws Ddl // analyze replication_num short replicationNum = FeConstants.default_replication_num; + String logReplicationNum = ""; try { boolean isReplicationNumSet = properties != null && properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM); + if (properties != null) { + logReplicationNum = properties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM); + } replicationNum = PropertyAnalyzer.analyzeReplicationNum(properties, replicationNum); if (isReplicationNumSet) { olapTable.setReplicationNum(replicationNum); } - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); + } catch (AnalysisException ex) { + throw new DdlException(String.format("%s table=%s, properties.replication_num=%s", + ex.getMessage(), olapTable.getName(), logReplicationNum)); } // set in memory @@ -2677,7 +2682,7 @@ private void createLakeTablets(LakeTable table, long partitionId, MaterializedIn } } - private void createOlapTablets(MaterializedIndex index, Replica.ReplicaState replicaState, + private void createOlapTablets(OlapTable table, MaterializedIndex index, Replica.ReplicaState replicaState, DistributionInfo distributionInfo, long version, short replicationNum, TabletMeta tabletMeta, Set tabletIdSet) throws DdlException { Preconditions.checkArgument(replicationNum > 0); @@ -2738,7 +2743,12 @@ private void createOlapTablets(MaterializedIndex index, Replica.ReplicaState rep chosenBackendIds = chosenBackendIdBySeq(replicationNum, tabletMeta.getStorageMedium()); } else { - chosenBackendIds = chosenBackendIdBySeq(replicationNum); + try { + chosenBackendIds = chosenBackendIdBySeq(replicationNum); + } catch (DdlException ex) { + throw new DdlException(String.format("%stable=%s, default_replication_num=%d", + ex.getMessage(), table.getName(), FeConstants.default_replication_num)); + } } backendsPerBucketSeq.add(chosenBackendIds); } else { @@ -2793,8 +2803,10 @@ private List chosenBackendIdBySeq(int replicationNum) throws DdlException systemInfoService.seqChooseBackendIds(replicationNum, true, true); if (CollectionUtils.isEmpty(chosenBackendIds)) { List backendIds = systemInfoService.getBackendIds(true); - throw new DdlException("Failed to find enough host in all backends. need: " + replicationNum + - ", Current alive backend is [" + Joiner.on(",").join(backendIds) + "]"); + throw new DdlException( + String.format("Table replication num should be less than of equal to the number of available BE nodes. " + + "You can change this default by setting the replication_num table properties. " + + "Current alive backend is [%s]. ", Joiner.on(",").join(backendIds))); } return chosenBackendIds; } From 2d665dc27086defd92ceb8b7baf4256a3abeedf4 Mon Sep 17 00:00:00 2001 From: liuyehcf <1559500551@qq.com> Date: Thu, 3 Aug 2023 08:45:08 +0800 Subject: [PATCH 19/23] [BugFix] Fix the wrong query state of failed query (#28500) ## How to reproduce ```sql -- enable profile set enable_profile=true; -- set memory limit to a small value to generate OOM deliberately SELECT /*+ SET_VAR(query_mem_limit='1')*/ COUNT(*) from lineitem; -- run same query normally SELECT COUNT(*) from lineitem; -- run failed query again SELECT /*+ SET_VAR(query_mem_limit='1')*/ COUNT(*) from lineitem; ``` You will see the state of the three query is: error, finished, running. ## Root cause All the queries within a connection share the single instance of QueryState, and `QueryState::reset()` will be called every time when query is finished. And some of the fields are not reset. and here's part of the cancel process listed down below, which shows that the queryState will not be set if the errorMessage is not empty, finally resulting to this problem. ```java private void cancelInternal(PPlanFragmentCancelReason cancelReason) { if (StringUtils.isEmpty(connectContext.getState().getErrorMessage())) { connectContext.getState().setError(cancelReason.toString()); } ... } ``` (cherry picked from commit fffdd7a01af000d82c0cba07753d9481c7fb19b1) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java --- .../src/main/java/com/starrocks/qe/QueryState.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java b/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java index 41351ef29921c6..ada02ad5072a47 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java @@ -70,10 +70,18 @@ public QueryState() { public void reset() { stateType = MysqlStateType.OK; + errorMessage = ""; errorCode = null; infoMessage = null; - serverStatus = 0; + errType = ErrType.OTHER_ERR; isQuery = false; +<<<<<<< HEAD +======= + affectedRows = 0; + warningRows = 0; + serverStatus = 0; + isFinished = false; +>>>>>>> fffdd7a01a ([BugFix] Fix the wrong query state of failed query (#28500)) } public MysqlStateType getStateType() { From a9e465235610735c689b1d8ac95cb0d1250cdde2 Mon Sep 17 00:00:00 2001 From: liuyehcf <1559500551@qq.com> Date: Thu, 3 Aug 2023 10:10:49 +0800 Subject: [PATCH 20/23] fix conflict Signed-off-by: liuyehcf <1559500551@qq.com> --- fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java b/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java index ada02ad5072a47..0fd1b439dbc758 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/QueryState.java @@ -75,13 +75,9 @@ public void reset() { infoMessage = null; errType = ErrType.OTHER_ERR; isQuery = false; -<<<<<<< HEAD -======= affectedRows = 0; warningRows = 0; serverStatus = 0; - isFinished = false; ->>>>>>> fffdd7a01a ([BugFix] Fix the wrong query state of failed query (#28500)) } public MysqlStateType getStateType() { From 9feb716a343a7ec1bbfc9711033a15b3b9d16d9b Mon Sep 17 00:00:00 2001 From: Seaven Date: Thu, 3 Aug 2023 16:03:56 +0800 Subject: [PATCH 21/23] [BugFix] Add now function (#28564) Fixes https://github.com/StarRocks/starrocks/pull/28262 Signed-off-by: Seaven --- .../starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java index aa7590e27c4013..d753b20790862a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorFunctions.java @@ -338,6 +338,7 @@ public static ConstantOperator fromUnixTime(ConstantOperator unixTime, ConstantO return dateFormat(dl, fmtLiteral); } + @ConstantFunction(name = "now", argTypes = {}, returnType = DATETIME) public static ConstantOperator now() { ConnectContext connectContext = ConnectContext.get(); LocalDateTime startTime = Instant.ofEpochMilli(connectContext.getStartTime()) From 6b358d6bc34c48f352454b10eafc7fa87af007d7 Mon Sep 17 00:00:00 2001 From: hellolilyliuyi <96421222+hellolilyliuyi@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:53:49 +0800 Subject: [PATCH 22/23] [Doc] updating pk table guarantees atomicity (#28590) (cherry picked from commit 7b5ef2ba6844bbe4f52d8bc5d5918462f05f3c70) --- docs/table_design/table_types/primary_key_table.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/table_design/table_types/primary_key_table.md b/docs/table_design/table_types/primary_key_table.md index 37be0447441ca9..1e4e4f56930d38 100644 --- a/docs/table_design/table_types/primary_key_table.md +++ b/docs/table_design/table_types/primary_key_table.md @@ -131,4 +131,6 @@ PROPERTIES("replication_num" = "3", ## What to do next -You can run a stream load, broker load, or routine load job to perform insert, update, or delete operations on all or individual columns of a table that uses the Primary Key table. For more information, see [Overview of data loading](../../loading/Loading_intro.md). +After table creation, you can run load jobs to load data into the Primary Key table. For more information about supported loading methods, see [Overview of data loading](../../loading/Loading_intro.md). + +If you need to update data in the Primary Key table, you can [run a load job](../../loading/Load_to_Primary_Key_tables.md) or execute a DML statement ([UPDATE](../../sql-reference/sql-statements/data-manipulation/UPDATE.md) or [DELETE](../../sql-reference/sql-statements/data-manipulation/DELETE.md)). Also, these update operations guarantee atomicity. From f0b62f4cfd0f7885999ef0ca803a1c4c2d405ccb Mon Sep 17 00:00:00 2001 From: packy92 <110370499+packy92@users.noreply.github.com> Date: Wed, 2 Aug 2023 10:48:03 +0800 Subject: [PATCH 23/23] [BugFix] remove unnecessary check in statistic calculate phase (backport #27646) Signed-off-by: packy92 --- .../operator/scalar/ColumnRefOperator.java | 14 ++++++------- .../ColumnBasicStatsCacheLoader.java | 14 ++++++++++++- .../sql/optimizer/statistics/ColumnDict.java | 3 ++- .../optimizer/statistics/ColumnStatistic.java | 20 ++++++++++++++++--- .../ExpressionStatisticCalculator.java | 4 +++- .../PredicateStatisticsCalculator.java | 7 ------- .../statistics/StatisticRangeValues.java | 17 ++-------------- .../sql/optimizer/statistics/Statistics.java | 13 ++++++++---- .../statistics/StatisticsCalculator.java | 15 +++----------- .../statistics/StatisticsCalculatorTest.java | 15 ++++++++++++++ 10 files changed, 70 insertions(+), 52 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ColumnRefOperator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ColumnRefOperator.java index eac572900e47a2..1e28410c9a1eb7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ColumnRefOperator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/ColumnRefOperator.java @@ -5,8 +5,10 @@ import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.operator.OperatorType; +import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.StringJoiner; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; @@ -93,16 +95,12 @@ public String toString() { return id + ": " + name; } - public static String toString(List columns) { - StringBuilder sb = new StringBuilder(); - int i = 0; + public static String toString(Collection columns) { + StringJoiner joiner = new StringJoiner("{", ", ", "}"); for (ColumnRefOperator column : columns) { - if (i++ != 0) { - sb.append(","); - } - sb.append(column); + joiner.add(column.toString()); } - return sb.toString(); + return joiner.toString(); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java index aca5bc477e03e5..1b851865153730 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnBasicStatsCacheLoader.java @@ -126,6 +126,7 @@ private ColumnStatistic convert2ColumnStatistics(TStatisticData statisticData) t ColumnStatistic.Builder builder = ColumnStatistic.builder(); double minValue = Double.NEGATIVE_INFINITY; double maxValue = Double.POSITIVE_INFINITY; + double distinctValues = statisticData.countDistinct; try { if (column.getPrimitiveType().isCharFamily()) { // do nothing @@ -158,9 +159,20 @@ private ColumnStatistic convert2ColumnStatistics(TStatisticData statisticData) t db.getFullName(), table.getName(), column.getName(), e.getMessage()); } + if (minValue > maxValue) { + LOG.warn("Min: {}, Max: {} values abnormal for db : {}, table : {}, column : {}", minValue, maxValue, + db.getFullName(), table.getName(), column.getName()); + minValue = Double.NEGATIVE_INFINITY; + maxValue = Double.POSITIVE_INFINITY; + } + + if (distinctValues <= 0) { + distinctValues = 1; + } + return builder.setMinValue(minValue). setMaxValue(maxValue). - setDistinctValuesCount(statisticData.countDistinct). + setDistinctValuesCount(distinctValues). setAverageRowSize(statisticData.dataSize / Math.max(statisticData.rowCount, 1)). setNullsFraction(statisticData.nullCount * 1.0 / Math.max(statisticData.rowCount, 1)). setRowCount(Math.max(statisticData.rowCount, 1)).build(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnDict.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnDict.java index 81dabab3b0dad9..76cdddb4025a91 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnDict.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnDict.java @@ -13,7 +13,8 @@ public final class ColumnDict { private long versionTime; public ColumnDict(ImmutableMap dict, long versionTime) { - Preconditions.checkState(dict.size() > 0 && dict.size() <= 256); + Preconditions.checkState(dict.size() > 0 && dict.size() <= 256, + "dict size %s is illegal", dict.size()); this.dict = dict; this.collectedVersionTime = versionTime; this.versionTime = versionTime; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnStatistic.java index 729f331aa9e6b3..adc612231876fc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnStatistic.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ColumnStatistic.java @@ -134,11 +134,25 @@ public static Builder buildFrom(String columnStatistic) { String typeString = endIndex == columnStatistic.length() - 1 ? "" : columnStatistic.substring(endIndex + 2); String[] valueArray = valueString.split(","); - Preconditions.checkState(valueArray.length == 5); + Preconditions.checkState(valueArray.length == 5, + "statistic value: %s is illegal", valueString); - Builder builder = new Builder(Double.parseDouble(valueArray[0]), Double.parseDouble(valueArray[1]), + double minValue = Double.parseDouble(valueArray[0]); + double maxValue = Double.parseDouble(valueArray[1]); + double distinctValues = Double.parseDouble(valueArray[4]); + + if (minValue > maxValue) { + minValue = Double.NEGATIVE_INFINITY; + maxValue = Double.POSITIVE_INFINITY; + } + + if (distinctValues <= 0) { + distinctValues = 1; + } + + Builder builder = new Builder(minValue, maxValue, Double.parseDouble(valueArray[2]), Double.parseDouble(valueArray[3]), - Double.parseDouble(valueArray[4])); + distinctValues); if (!typeString.isEmpty()) { builder.setType(StatisticType.valueOf(typeString)); } else if (builder.build().isUnknownValue()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ExpressionStatisticCalculator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ExpressionStatisticCalculator.java index 184ea14654b6bb..538c1e36983640 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ExpressionStatisticCalculator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/ExpressionStatisticCalculator.java @@ -144,7 +144,9 @@ public ColumnStatistic visitCastOperator(CastOperator cast, Void context) { public ColumnStatistic visitCall(CallOperator call, Void context) { List childrenColumnStatistics = call.getChildren().stream().map(child -> child.accept(this, context)).collect(Collectors.toList()); - Preconditions.checkState(childrenColumnStatistics.size() == call.getChildren().size()); + Preconditions.checkState(childrenColumnStatistics.size() == call.getChildren().size(), + "column statistics missing for expr: %s. column statistics: %s", + call, childrenColumnStatistics); if (childrenColumnStatistics.stream().anyMatch(ColumnStatistic::isUnknown) || inputStatistics.getColumnStatistics().values().stream().allMatch(ColumnStatistic::isUnknown)) { return ColumnStatistic.unknown(); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/PredicateStatisticsCalculator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/PredicateStatisticsCalculator.java index 67e122ffcf2fab..1873cbb2bb40ad 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/PredicateStatisticsCalculator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/PredicateStatisticsCalculator.java @@ -200,7 +200,6 @@ public Statistics visitBinaryPredicate(BinaryPredicateOperator predicate, Void c if (!checkNeedEvalEstimate(predicate)) { return statistics; } - Preconditions.checkState(predicate.getChildren().size() == 2); ScalarOperator leftChild = predicate.getChild(0); ScalarOperator rightChild = predicate.getChild(1); Preconditions.checkState(!(leftChild.isConstantRef() && rightChild.isConstantRef()), @@ -293,7 +292,6 @@ public Statistics visitCompoundPredicate(CompoundPredicateOperator predicate, Vo return StatisticsEstimateUtils.adjustStatisticsByRowCount(cumulativeStatistics, rowCount); } else { - Preconditions.checkState(predicate.getChildren().size() == 1); Statistics inputStatistics = predicate.getChild(0).accept(this, null); double rowCount = Math.max(0, statistics.getOutputRowCount() - inputStatistics.getOutputRowCount()); return StatisticsEstimateUtils.adjustStatisticsByRowCount( @@ -327,7 +325,6 @@ public Statistics visitConstant(ConstantOperator constant, Void context) { private ScalarOperator getChildForCastOperator(ScalarOperator operator) { if (operator instanceof CastOperator) { - Preconditions.checkState(operator.getChildren().size() == 1); operator = getChildForCastOperator(operator.getChild(0)); } return operator; @@ -350,15 +347,12 @@ public Statistics visitCompoundPredicate(CompoundPredicateOperator predicate, Vo } if (predicate.isAnd()) { - Preconditions.checkState(predicate.getChildren().size() == 2); Statistics leftStatistics = predicate.getChild(0).accept(this, null); Statistics andStatistics = predicate.getChild(1) .accept(new LargeOrCalculatingVisitor(leftStatistics), null); return StatisticsEstimateUtils.adjustStatisticsByRowCount(andStatistics, andStatistics.getOutputRowCount()); } else if (predicate.isOr()) { - Preconditions.checkState(predicate.getChildren().size() == 2); - List disjunctive = Utils.extractDisjunctive(predicate); Statistics baseStatistics = disjunctive.get(0).accept(this, null); double rowCount = baseStatistics.getOutputRowCount(); @@ -374,7 +368,6 @@ public Statistics visitCompoundPredicate(CompoundPredicateOperator predicate, Vo return StatisticsEstimateUtils.adjustStatisticsByRowCount(baseStatistics, rowCount); } else { - Preconditions.checkState(predicate.getChildren().size() == 1); Statistics inputStatistics = predicate.getChild(0).accept(this, null); double rowCount = Math.max(0, statistics.getOutputRowCount() - inputStatistics.getOutputRowCount()); return StatisticsEstimateUtils.adjustStatisticsByRowCount( diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticRangeValues.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticRangeValues.java index c9f4b831679114..a72afba32b2426 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticRangeValues.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticRangeValues.java @@ -2,18 +2,15 @@ package com.starrocks.sql.optimizer.statistics; -import com.google.common.base.Preconditions; - import java.util.Objects; +import javax.validation.constraints.NotNull; -import static com.google.common.base.Preconditions.checkArgument; import static java.lang.Double.NaN; import static java.lang.Double.isFinite; import static java.lang.Double.isInfinite; import static java.lang.Double.isNaN; import static java.lang.Math.max; import static java.lang.Math.min; -import static java.util.Objects.requireNonNull; // Calculate the cross range and ratio between column statistics public class StatisticRangeValues { @@ -22,16 +19,8 @@ public class StatisticRangeValues { private final double distinctValues; public StatisticRangeValues(double low, double high, double distinctValues) { - Preconditions.checkArgument( - low <= high || (isNaN(low) && isNaN(high)), - "low value must be less than or equal to high value or both values have to be NaN, got %s and %s respectively", - low, - high); this.low = low; this.high = high; - - checkArgument(distinctValues >= 0 || isNaN(distinctValues), - "Distinct values count should be non-negative, got: %s", distinctValues); this.distinctValues = distinctValues; } @@ -68,9 +57,7 @@ public double length() { } // Calculate the proportion of coverage between column statistic range - public double overlapPercentWith(StatisticRangeValues other) { - requireNonNull(other, "other is null"); - + public double overlapPercentWith(@NotNull StatisticRangeValues other) { if (this.isEmpty() || other.isEmpty()) { return 0.0; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/Statistics.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/Statistics.java index b7d488bf6295ca..5b9b6ccaff1e25 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/Statistics.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/Statistics.java @@ -2,8 +2,9 @@ package com.starrocks.sql.optimizer.statistics; -import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import com.starrocks.sql.common.ErrorType; +import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.statistic.StatsConstants; @@ -54,9 +55,13 @@ public double getComputeSize() { } public ColumnStatistic getColumnStatistic(ColumnRefOperator column) { - ColumnStatistic result = columnStatistics.get(column); - Preconditions.checkState(result != null); - return result; + if (columnStatistics.get(column) == null) { + throw new StarRocksPlannerException(ErrorType.INTERNAL_ERROR, + "only found column statistics: %s, but missing statistic of col: %s.", + ColumnRefOperator.toString(columnStatistics.keySet()), column); + } else { + return columnStatistics.get(column); + } } public Map getColumnStatistics() { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculator.java index 71225101f053cf..34d02005f44f60 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculator.java @@ -186,7 +186,6 @@ public Void visitOperator(Operator node, ExpressionContext context) { Projection projection = node.getProjection(); if (projection != null) { - Preconditions.checkState(projection.getCommonSubOperatorMap().isEmpty()); for (ColumnRefOperator columnRefOperator : projection.getColumnRefMap().keySet()) { ScalarOperator mapOperator = projection.getColumnRefMap().get(columnRefOperator); statisticsBuilder.addColumnStatistic(columnRefOperator, @@ -530,7 +529,6 @@ public Void visitLogicalProject(LogicalProjectOperator node, ExpressionContext c @Override public Void visitPhysicalProject(PhysicalProjectOperator node, ExpressionContext context) { - Preconditions.checkState(node.getCommonSubOperatorMap().isEmpty()); return computeProjectNode(context, node.getColumnRefMap()); } @@ -1262,14 +1260,6 @@ public Void visitPhysicalCTEConsume(PhysicalCTEConsumeOperator node, ExpressionC private Void computeCTEConsume(Operator node, ExpressionContext context, int cteId, Map columnRefMap) { - Optional produceStatisticsOp = optimizerContext.getCteContext().getCTEStatistics(cteId); - - // The statistics of producer and children are equal theoretically, but statistics of children - // plan maybe more accurate in actually - if (!produceStatisticsOp.isPresent() && (context.getChildrenStatistics().isEmpty() || - context.getChildrenStatistics().stream().anyMatch(Objects::isNull))) { - Preconditions.checkState(false, "Impossible cte statistics"); - } if (!context.getChildrenStatistics().isEmpty() && context.getChildStatistics(0) != null) { // use the statistics of children first @@ -1277,7 +1267,6 @@ private Void computeCTEConsume(Operator node, ExpressionContext context, int cte Projection projection = node.getProjection(); if (projection != null) { Statistics.Builder statisticsBuilder = Statistics.buildFrom(context.getStatistics()); - Preconditions.checkState(projection.getCommonSubOperatorMap().isEmpty()); for (ColumnRefOperator columnRefOperator : projection.getColumnRefMap().keySet()) { ScalarOperator mapOperator = projection.getColumnRefMap().get(columnRefOperator); statisticsBuilder.addColumnStatistic(columnRefOperator, @@ -1289,7 +1278,9 @@ private Void computeCTEConsume(Operator node, ExpressionContext context, int cte } // None children, may force CTE, use the statistics of producer - Preconditions.checkState(produceStatisticsOp.isPresent()); + Optional produceStatisticsOp = optimizerContext.getCteContext().getCTEStatistics(cteId); + Preconditions.checkState(produceStatisticsOp.isPresent(), + "cannot obtain cte statistics for %s", node); Statistics produceStatistics = produceStatisticsOp.get(); Statistics.Builder builder = Statistics.builder(); for (ColumnRefOperator ref : columnRefMap.keySet()) { diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java index 72fefcb5ddd1e6..89c0627f1bf404 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java @@ -14,6 +14,7 @@ import com.starrocks.common.FeConstants; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.optimizer.ExpressionContext; import com.starrocks.sql.optimizer.Group; import com.starrocks.sql.optimizer.GroupExpression; @@ -594,4 +595,18 @@ CompoundPredicateOperator.CompoundType.AND, eqOnPredicate1, new CompoundPredicat statisticsCalculator.estimatorStats(); Assert.assertEquals(expressionContext.getStatistics().getOutputRowCount(), 200000.0, 0.0001); } + + @Test + public void testNotFoundColumnStatistics() { + ColumnRefOperator v1 = columnRefFactory.create("v1", Type.INT, true); + ColumnRefOperator v2 = columnRefFactory.create("v2", Type.INT, true); + + ColumnRefOperator v3 = columnRefFactory.create("v3", Type.INT, true); + Statistics.Builder builder = Statistics.builder(); + builder.setOutputRowCount(10000); + builder.addColumnStatistics(ImmutableMap.of(v1, new ColumnStatistic(0, 100, 0, 10, 50))); + builder.addColumnStatistics(ImmutableMap.of(v2, new ColumnStatistic(0, 100, 0, 10, 50))); + Statistics statistics = builder.build(); + Assert.assertThrows(StarRocksPlannerException.class, () -> statistics.getColumnStatistic(v3)); + } }