From c76db5bc53cae0c722195ef38f68467755e35424 Mon Sep 17 00:00:00 2001 From: ABing <101158374+ABingHuang@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:06:31 +0800 Subject: [PATCH] [Bugfix] fix rewrite bug after insert new partition data (#20157) Fix rewrite failure after inserting new partition data. The bug reason is that the mv plan is cached, but the plan may change after ingestion with new partitions data. And the logic of partition predicate calculation for mv rewrite depends on scan node's selected partition id, which may change after ingestion. So it leads to invalid compensation predicate in mv rewrite, which caused an invalid rewritten plan. Fix it by: 1. disable partition prune rules during compile mv plan, which will keep partition predicates in mv plan, to avoid the problem of caching plan 2. remove mv partition predicate compensation logic because it is not necessary after step 1. and it also fixed the problem of redundant partition predicates after partition prune of mv. Signed-off-by: ABingHuang (cherry picked from commit ce97629f650ffb85eded065e7c6c5c347e1708f0) --- .../sql/optimizer/MvRewriteContext.java | 13 ++++ .../sql/optimizer/MvRewritePreprocessor.java | 48 +----------- .../materialization/MVPartitionPruner.java | 48 ++++++++++-- .../MaterializedViewRewriter.java | 74 +++++++++++++------ .../materialization/MvUtils.java | 7 +- .../rule/BaseMaterializedViewRewriteRule.java | 7 +- .../planner/MaterializedViewTest.java | 3 +- .../planner/MaterializedViewTestBase.java | 7 +- .../MaterializedViewWithPartitionTest.java | 44 +++-------- .../MvRewriteOptimizationTest.java | 6 +- 10 files changed, 135 insertions(+), 122 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java index 59e688603bff1..dc8e801340dec 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java @@ -16,6 +16,7 @@ package com.starrocks.sql.optimizer; import com.starrocks.catalog.Table; +import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter; import com.starrocks.sql.optimizer.rule.transformation.materialization.PredicateSplit; @@ -32,6 +33,10 @@ public class MvRewriteContext { private final ReplaceColumnRefRewriter queryColumnRefRewriter; private final PredicateSplit queryPredicateSplit; + // mv's partition and distribution related conjunct predicate, + // used to prune partitions and buckets of scan mv operator after rewrite + private ScalarOperator mvPruneConjunct; + public MvRewriteContext( MaterializationContext materializationContext, List queryTables, @@ -64,4 +69,12 @@ public ReplaceColumnRefRewriter getQueryColumnRefRewriter() { public PredicateSplit getQueryPredicateSplit() { return queryPredicateSplit; } + + public ScalarOperator getMvPruneConjunct() { + return mvPruneConjunct; + } + + public void setMvPruneConjunct(ScalarOperator mvPruneConjunct) { + this.mvPruneConjunct = mvPruneConjunct; + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java index fae349a3d8ebb..37608831e457d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java @@ -20,14 +20,11 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.sql.ast.PartitionNames; import com.starrocks.sql.optimizer.base.ColumnRefFactory; -import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.DistributionSpec; import com.starrocks.sql.optimizer.base.HashDistributionDesc; import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; -import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; -import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; import org.apache.logging.log4j.LogManager; @@ -176,7 +173,6 @@ private LogicalOlapScanOperator createScanMvOperator(MaterializationContext mvCo final Map columnMetaToColRefMap = columnMetaToColRefMapBuilder.build(); // construct distribution - final Set mvPartitionDistributionColumnRef = Sets.newHashSet(); DistributionInfo distributionInfo = mv.getDefaultDistributionInfo(); // only hash distribution is supported Preconditions.checkState(distributionInfo instanceof HashDistributionInfo); @@ -204,54 +200,12 @@ private LogicalOlapScanOperator createScanMvOperator(MaterializationContext mvCo } final PartitionNames partitionNames = new PartitionNames(false, selectedPartitionNames); - // NOTE: - // - To partition/distribution prune, need filter predicates that belong to MV. - // - Those predicates are only used for partition/distribution pruning and don't affect the real - // query compute. - // - after partition/distribution pruning, those predicates should be removed from mv rewrite result. - final OptExpression mvExpression = mvContext.getMvExpression(); - final List conjuncts = MvUtils.getAllPredicates(mvExpression); - final ColumnRefSet mvOutputColumnRefSet = mvExpression.getOutputColumns(); - final List mvConjuncts = Lists.newArrayList(); - - // Construct partition/distribution key column refs to filter conjunctions which need to retain. - Set mvPruneKeyColNames = Sets.newHashSet(); - distributedColumns.stream().forEach(distKey -> mvPruneKeyColNames.add(distKey.getName())); - mv.getPartitionNames().stream().forEach(partName -> mvPruneKeyColNames.add(partName)); - final Set mvPruneColumnIdSet = mvOutputColumnRefSet.getStream().map( - id -> mvContext.getMvColumnRefFactory().getColumnRef(id)) - .filter(colRef -> mvPruneKeyColNames.contains(colRef.getName())) - .map(colRef -> colRef.getId()) - .collect(Collectors.toSet()); - // Case1: keeps original predicates which belong to MV table(which are not pruned after mv's partition pruning) - for (ScalarOperator conj : conjuncts) { - // ignore binary predicates which cannot be used for pruning. - if (conj instanceof BinaryPredicateOperator) { - BinaryPredicateOperator conjOp = (BinaryPredicateOperator) conj; - if (conjOp.getChild(0).isColumnRef() && conjOp.getChild(1).isColumnRef()) { - continue; - } - } - final List conjColumnRefOperators = - Utils.extractColumnRef(conj).stream().map(ref -> ref.getId()).collect(Collectors.toList()); - if (mvPruneColumnIdSet.containsAll(conjColumnRefOperators)) { - mvConjuncts.add(conj); - } - } - // Case2: compensated partition predicates which are pruned after mv's partition pruning. - // Compensate partition predicates and add them into mv predicate. - final ScalarOperator mvPartitionPredicate = - MvUtils.compensatePartitionPredicate(mvExpression, mvContext.getMvColumnRefFactory()); - if (!ConstantOperator.TRUE.equals(mvPartitionPredicate)) { - mvConjuncts.add(mvPartitionPredicate); - } - return new LogicalOlapScanOperator(mv, colRefToColumnMetaMapBuilder.build(), columnMetaToColRefMap, DistributionSpec.createHashDistributionSpec(hashDistributionDesc), Operator.DEFAULT_LIMIT, - Utils.compoundAnd(mvConjuncts), + null, mv.getBaseIndexId(), selectPartitionIds, partitionNames, diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java index f7605ee6a3b51..3e0427bab09f5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java @@ -15,9 +15,11 @@ package com.starrocks.sql.optimizer.rule.transformation.materialization; import com.google.common.collect.Lists; +import com.starrocks.sql.optimizer.MvRewriteContext; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptExpressionVisitor; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorBuilderFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalDeltaLakeScanOperator; @@ -28,6 +30,7 @@ import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; +import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.OptDistributionPruner; import com.starrocks.sql.optimizer.rewrite.OptExternalPartitionPruner; import com.starrocks.sql.optimizer.rewrite.OptOlapPartitionPruner; @@ -35,14 +38,21 @@ import java.util.List; public class MVPartitionPruner { + private final OptimizerContext optimizerContext; + private final MvRewriteContext mvRewriteContext; - public OptExpression prunePartition(OptimizerContext context, OptExpression queryExpression) { - return queryExpression.getOp().accept(new MVPartitionPrunerVisitor(), queryExpression, context); + public MVPartitionPruner(OptimizerContext optimizerContext, MvRewriteContext mvRewriteContext) { + this.optimizerContext = optimizerContext; + this.mvRewriteContext = mvRewriteContext; } - private class MVPartitionPrunerVisitor extends OptExpressionVisitor { + public OptExpression prunePartition(OptExpression queryExpression) { + return queryExpression.getOp().accept(new MVPartitionPrunerVisitor(), queryExpression, null); + } + + private class MVPartitionPrunerVisitor extends OptExpressionVisitor { @Override - public OptExpression visitLogicalTableScan(OptExpression optExpression, OptimizerContext context) { + public OptExpression visitLogicalTableScan(OptExpression optExpression, Void context) { LogicalScanOperator scanOperator = optExpression.getOp().cast(); if (scanOperator instanceof LogicalOlapScanOperator) { @@ -56,6 +66,18 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Optimize .setPrunedPartitionPredicates(Lists.newArrayList()) .setSelectedPartitionId(Lists.newArrayList()) .setSelectedTabletId(Lists.newArrayList()); + + // for mv: select c1, c3, c2 from test_base_part where c3 < 2000 and c1 = 1, + // which c3 is partition column and c1 is distribution column. + // we should add predicate c3 < 2000 and c1 = 1 into scan operator to do pruning + boolean isAddMvPrunePredicate = scanOperator.getTable().isMaterializedView() + && scanOperator.getTable().getId() == mvRewriteContext.getMaterializationContext().getMv().getId() + && mvRewriteContext.getMvPruneConjunct() != null; + if (isAddMvPrunePredicate) { + ScalarOperator originPredicate = scanOperator.getPredicate(); + ScalarOperator newPredicate = Utils.compoundAnd(originPredicate, mvRewriteContext.getMvPruneConjunct()); + builder.setPredicate(newPredicate); + } LogicalOlapScanOperator copiedOlapScanOperator = builder.build(); // prune partition @@ -67,9 +89,19 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Optimize List selectedTabletIds = OptDistributionPruner.pruneTabletIds(copiedOlapScanOperator, prunedOlapScanOperator.getSelectedPartitionId()); + ScalarOperator scanPredicate = prunedOlapScanOperator.getPredicate(); + if (isAddMvPrunePredicate) { + List originConjuncts = Utils.extractConjuncts(scanOperator.getPredicate()); + List pruneConjuncts = Utils.extractConjuncts(mvRewriteContext.getMvPruneConjunct()); + pruneConjuncts.removeAll(originConjuncts); + List currentConjuncts = Utils.extractConjuncts(prunedOlapScanOperator.getPredicate()); + currentConjuncts.removeAll(pruneConjuncts); + scanPredicate = Utils.compoundAnd(currentConjuncts); + } + LogicalOlapScanOperator.Builder rewrittenBuilder = new LogicalOlapScanOperator.Builder(); scanOperator = rewrittenBuilder.withOperator(prunedOlapScanOperator) - .setPredicate(MvUtils.canonizePredicate(prunedOlapScanOperator.getPredicate())) + .setPredicate(MvUtils.canonizePredicate(scanPredicate)) .setSelectedTabletId(selectedTabletIds) .build(); } else if (scanOperator instanceof LogicalHiveScanOperator || @@ -81,16 +113,16 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Optimize Operator.Builder builder = OperatorBuilderFactory.build(scanOperator); LogicalScanOperator copiedScanOperator = (LogicalScanOperator) builder.withOperator(scanOperator).build(); - scanOperator = OptExternalPartitionPruner.prunePartitions(context, + scanOperator = OptExternalPartitionPruner.prunePartitions(optimizerContext, copiedScanOperator); } return OptExpression.create(scanOperator); } - public OptExpression visit(OptExpression optExpression, OptimizerContext context) { + public OptExpression visit(OptExpression optExpression, Void context) { List children = Lists.newArrayList(); for (int i = 0; i < optExpression.arity(); ++i) { - children.add(optExpression.inputAt(i).getOp().accept(this, optExpression.inputAt(i), context)); + children.add(optExpression.inputAt(i).getOp().accept(this, optExpression.inputAt(i), null)); } return OptExpression.create(optExpression.getOp(), children); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java index 45125b8d6c8de..6700d5bf14722 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java @@ -27,8 +27,11 @@ import com.google.common.graph.MutableGraph; import com.starrocks.analysis.JoinOperator; import com.starrocks.catalog.Column; +import com.starrocks.catalog.DistributionInfo; import com.starrocks.catalog.ForeignKeyConstraint; +import com.starrocks.catalog.HashDistributionInfo; import com.starrocks.catalog.KeysType; +import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; import com.starrocks.catalog.UniqueConstraint; @@ -155,22 +158,9 @@ public OptExpression rewrite() { final ColumnRefFactory mvColumnRefFactory = materializationContext.getMvColumnRefFactory(); final ReplaceColumnRefRewriter mvColumnRefRewriter = MvUtils.getReplaceColumnRefWriter(mvExpression, mvColumnRefFactory); - // Compensate partition predicates and add them into mv predicate, - // eg: c3 is partition column - // MV : select c1, c3, c2 from test_base_part where c3 < 2000 - // Query : select c1, c3, c2 from test_base_part - // `c3 < 2000` is missed after partition pruning, so `mvPredicate` must add `mvPartitionPredicate`, - // otherwise query above may be rewritten by mv. - final ScalarOperator mvPartitionPredicate = - MvUtils.compensatePartitionPredicate(mvExpression, mvColumnRefFactory); - if (mvPartitionPredicate == null) { - return null; - } + ScalarOperator mvPredicate = MvUtils.rewriteOptExprCompoundPredicate(mvExpression, mvColumnRefRewriter); - if (!ConstantOperator.TRUE.equals(mvPartitionPredicate)) { - mvPredicate = MvUtils.canonizePredicate(Utils.compoundAnd(mvPredicate, mvPartitionPredicate)); - } if (materializationContext.getMvPartialPartitionPredicate() != null) { // add latest partition predicate to mv predicate ScalarOperator rewritten = mvColumnRefRewriter.rewrite(materializationContext.getMvPartialPartitionPredicate()); @@ -222,7 +212,12 @@ public OptExpression rewrite() { materializationContext.getMvColumnRefFactory(), mvColumnRefRewriter, materializationContext.getOutputMapping(), queryColumnSet); + // collect partition and distribution related predicates in mv + // used to prune partition and buckets after mv rewrite + ScalarOperator mvPrunePredicate = collectMvPrunePredicate(materializationContext); + for (BiMap relationIdMapping : relationIdMappings) { + mvRewriteContext.setMvPruneConjunct(mvPrunePredicate); rewriteContext.setQueryToMvRelationIdMapping(relationIdMapping); // for view delta, should add compensation join columns to query ec @@ -502,6 +497,45 @@ private boolean isJoinMatch(OptExpression queryExpression, } } + private ScalarOperator collectMvPrunePredicate(MaterializationContext mvContext) { + final OptExpression mvExpression = mvContext.getMvExpression(); + final List conjuncts = MvUtils.getAllPredicates(mvExpression); + final ColumnRefSet mvOutputColumnRefSet = mvExpression.getOutputColumns(); + // conjuncts related to partition and distribution + final List mvPrunePredicates = Lists.newArrayList(); + + // Construct partition/distribution key column refs to filter conjunctions which need to retain. + Set mvPruneKeyColNames = Sets.newHashSet(); + MaterializedView mv = mvContext.getMv(); + DistributionInfo distributionInfo = mv.getDefaultDistributionInfo(); + // only hash distribution is supported + Preconditions.checkState(distributionInfo instanceof HashDistributionInfo); + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + List distributedColumns = hashDistributionInfo.getDistributionColumns(); + distributedColumns.stream().forEach(distKey -> mvPruneKeyColNames.add(distKey.getName())); + mv.getPartitionColumnNames().stream().forEach(partName -> mvPruneKeyColNames.add(partName)); + final Set mvPruneColumnIdSet = mvOutputColumnRefSet.getStream().map( + id -> mvContext.getMvColumnRefFactory().getColumnRef(id)) + .filter(colRef -> mvPruneKeyColNames.contains(colRef.getName())) + .map(colRef -> colRef.getId()) + .collect(Collectors.toSet()); + for (ScalarOperator conj : conjuncts) { + // ignore binary predicates which cannot be used for pruning. + if (conj instanceof BinaryPredicateOperator) { + BinaryPredicateOperator conjOp = (BinaryPredicateOperator) conj; + if (conjOp.getChild(0).isVariable() && conjOp.getChild(1).isVariable()) { + continue; + } + } + final List conjColumnRefOperators = + Utils.extractColumnRef(conj).stream().map(ref -> ref.getId()).collect(Collectors.toList()); + if (mvPruneColumnIdSet.containsAll(conjColumnRefOperators)) { + mvPrunePredicates.add(conj); + } + } + return Utils.compoundAnd(mvPrunePredicates); + } + private OptExpression tryRewriteForRelationMapping(RewriteContext rewriteContext) { // the rewritten expression to replace query // should copy the op because the op will be modified and reused @@ -512,13 +546,11 @@ private OptExpression tryRewriteForRelationMapping(RewriteContext rewriteContext // Rewrite original mv's predicates into query if needed. final ColumnRewriter columnRewriter = new ColumnRewriter(rewriteContext); final Map mvColumnRefToScalarOp = rewriteContext.getMVColumnRefToScalarOp(); - ScalarOperator mvOriginalPredicates = mvScanOperator.getPredicate(); - if (mvOriginalPredicates != null && !ConstantOperator.TRUE.equals(mvOriginalPredicates)) { - mvOriginalPredicates = rewriteMVCompensationExpression(rewriteContext, columnRewriter, - mvColumnRefToScalarOp, mvOriginalPredicates, false); - if (!ConstantOperator.TRUE.equals(mvOriginalPredicates)) { - mvScanBuilder.setPredicate(mvOriginalPredicates); - } + if (mvRewriteContext.getMvPruneConjunct() != null + && !ConstantOperator.TRUE.equals(mvRewriteContext.getMvPruneConjunct())) { + ScalarOperator rewrittenPrunePredicate = rewriteMVCompensationExpression(rewriteContext, columnRewriter, + mvColumnRefToScalarOp, mvRewriteContext.getMvPruneConjunct(), false); + mvRewriteContext.setMvPruneConjunct(MvUtils.canonizePredicate(rewrittenPrunePredicate)); } OptExpression mvScanOptExpression = OptExpression.create(mvScanBuilder.build()); deriveLogicalProperty(mvScanOptExpression); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index 2bbc5c4930251..d2c2e163f6f19 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -320,6 +320,7 @@ public static Pair getRuleOptimizedLogicalPlan(Strin new RelationTransformer(columnRefFactory, connectContext).transformWithSelectLimit(query); // optimize the sql by rule and disable rule based materialized view rewrite OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); + optimizerConfig.disableRuleSet(RuleSetType.PARTITION_PRUNE); optimizerConfig.disableRuleSet(RuleSetType.SINGLE_TABLE_MV_REWRITE); Optimizer optimizer = new Optimizer(optimizerConfig); OptExpression optimizedPlan = optimizer.optimize( @@ -382,12 +383,14 @@ private static void getAllPredicates(OptExpression root, List pr // Ignore aggregation predicates, because aggregation predicates should be rewritten after // aggregation functions' rewrite and should not be pushed down into mv scan operator. if (operator.getPredicate() != null && !(operator instanceof LogicalAggregationOperator)) { - predicates.add(operator.getPredicate()); + List conjuncts = Utils.extractConjuncts(operator.getPredicate()); + predicates.addAll(conjuncts); } if (operator instanceof LogicalJoinOperator) { LogicalJoinOperator joinOperator = (LogicalJoinOperator) operator; if (joinOperator.getOnPredicate() != null) { - predicates.add(joinOperator.getOnPredicate()); + List conjuncts = Utils.extractConjuncts(joinOperator.getOnPredicate()); + predicates.addAll(conjuncts); } } for (OptExpression child : root.getInputs()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java index 5d3a13b7cbb3d..244168a5c3996 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java @@ -88,7 +88,7 @@ public List transform(OptExpression queryExpression, OptimizerCon MaterializedViewRewriter mvRewriter = getMaterializedViewRewrite(mvRewriteContext); OptExpression candidate = mvRewriter.rewrite(); if (candidate != null) { - candidate = postRewriteMV(context, candidate); + candidate = postRewriteMV(context, mvRewriteContext, candidate); if (queryExpression.getGroupExpression() != null) { int currentRootGroupId = queryExpression.getGroupExpression().getGroup().getId(); mvContext.addMatchedGroup(currentRootGroupId); @@ -106,12 +106,13 @@ public List transform(OptExpression queryExpression, OptimizerCon * 2. partition prune * 3. bucket prune */ - private OptExpression postRewriteMV(OptimizerContext context, OptExpression candidate) { + private OptExpression postRewriteMV( + OptimizerContext optimizerContext, MvRewriteContext mvRewriteContext, OptExpression candidate) { if (candidate == null) { return null; } candidate = new MVColumnPruner().pruneColumns(candidate); - candidate = new MVPartitionPruner().prunePartition(context, candidate); + candidate = new MVPartitionPruner(optimizerContext, mvRewriteContext).prunePartition(candidate); return candidate; } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java index 0245efb75932e..4e59dfb6b1506 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java @@ -1543,8 +1543,7 @@ public void testViewDeltaJoinUKFK5() { testRewriteOK(mv, query) .contains("0:OlapScanNode\n" + " TABLE: mv0\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 7: empid = 1"); + " PREAGGREGATION: ON\n"); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java index bcb5c88a7ae5e..c12d7112b91a6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java @@ -175,7 +175,12 @@ public MVRewriteChecker nonMatch() { } public MVRewriteChecker contains(String expect) { - Assert.assertTrue(this.rewritePlan.contains(expect)); + boolean contained = this.rewritePlan.contains(expect); + if (!contained) { + LOG.warn("rewritePlan: \n{}", rewritePlan); + LOG.warn("expect: \n{}", expect); + } + Assert.assertTrue(contained); return this; } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java index 952de5274095b..aafd5d3f11251 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java @@ -14,13 +14,10 @@ package com.starrocks.planner; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.BeforeClass; import org.junit.Test; public class MaterializedViewWithPartitionTest extends MaterializedViewTestBase { - private static final Logger LOG = LogManager.getLogger(MaterializedViewWithPartitionTest.class); @BeforeClass public static void setUp() throws Exception { @@ -127,10 +124,8 @@ public void testPartitionPrune_SingleTable2() throws Exception { .contains("UNION") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 6: c3 < 2000\n" + " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2") + " rollup: partial_mv_6") .contains("TABLE: test_base_part\n" + " PREAGGREGATION: ON\n" + " partitions=1/5\n" + @@ -143,10 +138,7 @@ public void testPartitionPrune_SingleTable2() throws Exception { .contains("partial_mv_6") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 6: c3 <= 999, 6: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 6: c3 <= 999"); starRocksAssert.dropMaterializedView("partial_mv_6"); } @@ -338,10 +330,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 2000") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 11: c3 < 2000\n" + " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " rollup: partial_mv_6"); // test query delta testRewriteOK( @@ -351,10 +341,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 2000 and t2.c3 > 100;") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 11: c3 <= 1999, 11: c3 >= 101, 11: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 11: c3 <= 1999, 11: c3 >= 101\n" + + " partitions=1/1"); // test query delta testRewriteOK( @@ -364,10 +352,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 1000;") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 11: c3 <= 999, 11: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 11: c3 <= 999\n" + + " partitions=1/1"); // test query delta, agg + join testRewriteOK( @@ -377,10 +363,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 1000;") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 12: c3 <= 999, 12: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 12: c3 <= 999\n" + + " partitions=1/1"); // test union all // TODO: MV can be rewritten but cannot bingo(because cost?)! @@ -435,21 +419,15 @@ public void testBucketPrune_SingleTable1() throws Exception { .contains("UNION") .contains("TABLE: partial_mv_7\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 5: c1 = 1\n" + " partitions=4/5\n" + " rollup: partial_mv_7\n" + " tabletRatio=4/8") - .contains("TABLE: test_base_part\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: (8: c1 != 1) OR (10: c3 >= 2000)\n" + - " partitions=5/5\n" + - " rollup: test_base_part"); + .contains("TABLE: test_base_part"); // match all testRewriteOK("select c1, c3, c2 from test_base_part where c3 < 2000 and c1 = 1") .contains("TABLE: partial_mv_7\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 5: c1 = 1\n" + " partitions=4/5\n" + " rollup: partial_mv_7\n" + " tabletRatio=4/8"); @@ -458,7 +436,7 @@ public void testBucketPrune_SingleTable1() throws Exception { testRewriteOK("select c1, c3, c2 from test_base_part where c3 < 1000 and c1 = 1") .contains("TABLE: partial_mv_7\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 6: c3 <= 999, 5: c1 = 1\n" + + " PREDICATES: 6: c3 <= 999\n" + " partitions=3/5\n" + " rollup: partial_mv_7\n" + " tabletRatio=3/6"); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index c73fff5064f8f..b1a9e9692e8de 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -1738,11 +1738,7 @@ public void testPartialPartition() throws Exception { " select c1, c3, c2 from test_base_part where c3 < 2000 and c1 = 1;"); String query11 = "select c1, c3, c2 from test_base_part"; String plan11 = getFragmentPlan(query11); - PlanTestBase.assertContains(plan11, "partial_mv_7", "UNION", "TABLE: test_base_part\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: (8: c1 != 1) OR (10: c3 >= 2000)\n" + - " partitions=6/6\n" + - " rollup: test_base_part"); + PlanTestBase.assertContains(plan11, "partial_mv_7", "UNION", "TABLE: test_base_part"); dropMv("test", "partial_mv_7"); createAndRefreshMv("test", "partial_mv_8", "create materialized view partial_mv_8" +