diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java index 59e688603bff1..dc8e801340dec 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewriteContext.java @@ -16,6 +16,7 @@ package com.starrocks.sql.optimizer; import com.starrocks.catalog.Table; +import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter; import com.starrocks.sql.optimizer.rule.transformation.materialization.PredicateSplit; @@ -32,6 +33,10 @@ public class MvRewriteContext { private final ReplaceColumnRefRewriter queryColumnRefRewriter; private final PredicateSplit queryPredicateSplit; + // mv's partition and distribution related conjunct predicate, + // used to prune partitions and buckets of scan mv operator after rewrite + private ScalarOperator mvPruneConjunct; + public MvRewriteContext( MaterializationContext materializationContext, List queryTables, @@ -64,4 +69,12 @@ public ReplaceColumnRefRewriter getQueryColumnRefRewriter() { public PredicateSplit getQueryPredicateSplit() { return queryPredicateSplit; } + + public ScalarOperator getMvPruneConjunct() { + return mvPruneConjunct; + } + + public void setMvPruneConjunct(ScalarOperator mvPruneConjunct) { + this.mvPruneConjunct = mvPruneConjunct; + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java index fae349a3d8ebb..37608831e457d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java @@ -20,14 +20,11 @@ import com.starrocks.qe.ConnectContext; import com.starrocks.sql.ast.PartitionNames; import com.starrocks.sql.optimizer.base.ColumnRefFactory; -import com.starrocks.sql.optimizer.base.ColumnRefSet; import com.starrocks.sql.optimizer.base.DistributionSpec; import com.starrocks.sql.optimizer.base.HashDistributionDesc; import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; -import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; -import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; import org.apache.logging.log4j.LogManager; @@ -176,7 +173,6 @@ private LogicalOlapScanOperator createScanMvOperator(MaterializationContext mvCo final Map columnMetaToColRefMap = columnMetaToColRefMapBuilder.build(); // construct distribution - final Set mvPartitionDistributionColumnRef = Sets.newHashSet(); DistributionInfo distributionInfo = mv.getDefaultDistributionInfo(); // only hash distribution is supported Preconditions.checkState(distributionInfo instanceof HashDistributionInfo); @@ -204,54 +200,12 @@ private LogicalOlapScanOperator createScanMvOperator(MaterializationContext mvCo } final PartitionNames partitionNames = new PartitionNames(false, selectedPartitionNames); - // NOTE: - // - To partition/distribution prune, need filter predicates that belong to MV. - // - Those predicates are only used for partition/distribution pruning and don't affect the real - // query compute. - // - after partition/distribution pruning, those predicates should be removed from mv rewrite result. - final OptExpression mvExpression = mvContext.getMvExpression(); - final List conjuncts = MvUtils.getAllPredicates(mvExpression); - final ColumnRefSet mvOutputColumnRefSet = mvExpression.getOutputColumns(); - final List mvConjuncts = Lists.newArrayList(); - - // Construct partition/distribution key column refs to filter conjunctions which need to retain. - Set mvPruneKeyColNames = Sets.newHashSet(); - distributedColumns.stream().forEach(distKey -> mvPruneKeyColNames.add(distKey.getName())); - mv.getPartitionNames().stream().forEach(partName -> mvPruneKeyColNames.add(partName)); - final Set mvPruneColumnIdSet = mvOutputColumnRefSet.getStream().map( - id -> mvContext.getMvColumnRefFactory().getColumnRef(id)) - .filter(colRef -> mvPruneKeyColNames.contains(colRef.getName())) - .map(colRef -> colRef.getId()) - .collect(Collectors.toSet()); - // Case1: keeps original predicates which belong to MV table(which are not pruned after mv's partition pruning) - for (ScalarOperator conj : conjuncts) { - // ignore binary predicates which cannot be used for pruning. - if (conj instanceof BinaryPredicateOperator) { - BinaryPredicateOperator conjOp = (BinaryPredicateOperator) conj; - if (conjOp.getChild(0).isColumnRef() && conjOp.getChild(1).isColumnRef()) { - continue; - } - } - final List conjColumnRefOperators = - Utils.extractColumnRef(conj).stream().map(ref -> ref.getId()).collect(Collectors.toList()); - if (mvPruneColumnIdSet.containsAll(conjColumnRefOperators)) { - mvConjuncts.add(conj); - } - } - // Case2: compensated partition predicates which are pruned after mv's partition pruning. - // Compensate partition predicates and add them into mv predicate. - final ScalarOperator mvPartitionPredicate = - MvUtils.compensatePartitionPredicate(mvExpression, mvContext.getMvColumnRefFactory()); - if (!ConstantOperator.TRUE.equals(mvPartitionPredicate)) { - mvConjuncts.add(mvPartitionPredicate); - } - return new LogicalOlapScanOperator(mv, colRefToColumnMetaMapBuilder.build(), columnMetaToColRefMap, DistributionSpec.createHashDistributionSpec(hashDistributionDesc), Operator.DEFAULT_LIMIT, - Utils.compoundAnd(mvConjuncts), + null, mv.getBaseIndexId(), selectPartitionIds, partitionNames, diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java index f7605ee6a3b51..3e0427bab09f5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVPartitionPruner.java @@ -15,9 +15,11 @@ package com.starrocks.sql.optimizer.rule.transformation.materialization; import com.google.common.collect.Lists; +import com.starrocks.sql.optimizer.MvRewriteContext; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptExpressionVisitor; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.Utils; import com.starrocks.sql.optimizer.operator.Operator; import com.starrocks.sql.optimizer.operator.OperatorBuilderFactory; import com.starrocks.sql.optimizer.operator.logical.LogicalDeltaLakeScanOperator; @@ -28,6 +30,7 @@ import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator; +import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.OptDistributionPruner; import com.starrocks.sql.optimizer.rewrite.OptExternalPartitionPruner; import com.starrocks.sql.optimizer.rewrite.OptOlapPartitionPruner; @@ -35,14 +38,21 @@ import java.util.List; public class MVPartitionPruner { + private final OptimizerContext optimizerContext; + private final MvRewriteContext mvRewriteContext; - public OptExpression prunePartition(OptimizerContext context, OptExpression queryExpression) { - return queryExpression.getOp().accept(new MVPartitionPrunerVisitor(), queryExpression, context); + public MVPartitionPruner(OptimizerContext optimizerContext, MvRewriteContext mvRewriteContext) { + this.optimizerContext = optimizerContext; + this.mvRewriteContext = mvRewriteContext; } - private class MVPartitionPrunerVisitor extends OptExpressionVisitor { + public OptExpression prunePartition(OptExpression queryExpression) { + return queryExpression.getOp().accept(new MVPartitionPrunerVisitor(), queryExpression, null); + } + + private class MVPartitionPrunerVisitor extends OptExpressionVisitor { @Override - public OptExpression visitLogicalTableScan(OptExpression optExpression, OptimizerContext context) { + public OptExpression visitLogicalTableScan(OptExpression optExpression, Void context) { LogicalScanOperator scanOperator = optExpression.getOp().cast(); if (scanOperator instanceof LogicalOlapScanOperator) { @@ -56,6 +66,18 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Optimize .setPrunedPartitionPredicates(Lists.newArrayList()) .setSelectedPartitionId(Lists.newArrayList()) .setSelectedTabletId(Lists.newArrayList()); + + // for mv: select c1, c3, c2 from test_base_part where c3 < 2000 and c1 = 1, + // which c3 is partition column and c1 is distribution column. + // we should add predicate c3 < 2000 and c1 = 1 into scan operator to do pruning + boolean isAddMvPrunePredicate = scanOperator.getTable().isMaterializedView() + && scanOperator.getTable().getId() == mvRewriteContext.getMaterializationContext().getMv().getId() + && mvRewriteContext.getMvPruneConjunct() != null; + if (isAddMvPrunePredicate) { + ScalarOperator originPredicate = scanOperator.getPredicate(); + ScalarOperator newPredicate = Utils.compoundAnd(originPredicate, mvRewriteContext.getMvPruneConjunct()); + builder.setPredicate(newPredicate); + } LogicalOlapScanOperator copiedOlapScanOperator = builder.build(); // prune partition @@ -67,9 +89,19 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Optimize List selectedTabletIds = OptDistributionPruner.pruneTabletIds(copiedOlapScanOperator, prunedOlapScanOperator.getSelectedPartitionId()); + ScalarOperator scanPredicate = prunedOlapScanOperator.getPredicate(); + if (isAddMvPrunePredicate) { + List originConjuncts = Utils.extractConjuncts(scanOperator.getPredicate()); + List pruneConjuncts = Utils.extractConjuncts(mvRewriteContext.getMvPruneConjunct()); + pruneConjuncts.removeAll(originConjuncts); + List currentConjuncts = Utils.extractConjuncts(prunedOlapScanOperator.getPredicate()); + currentConjuncts.removeAll(pruneConjuncts); + scanPredicate = Utils.compoundAnd(currentConjuncts); + } + LogicalOlapScanOperator.Builder rewrittenBuilder = new LogicalOlapScanOperator.Builder(); scanOperator = rewrittenBuilder.withOperator(prunedOlapScanOperator) - .setPredicate(MvUtils.canonizePredicate(prunedOlapScanOperator.getPredicate())) + .setPredicate(MvUtils.canonizePredicate(scanPredicate)) .setSelectedTabletId(selectedTabletIds) .build(); } else if (scanOperator instanceof LogicalHiveScanOperator || @@ -81,16 +113,16 @@ public OptExpression visitLogicalTableScan(OptExpression optExpression, Optimize Operator.Builder builder = OperatorBuilderFactory.build(scanOperator); LogicalScanOperator copiedScanOperator = (LogicalScanOperator) builder.withOperator(scanOperator).build(); - scanOperator = OptExternalPartitionPruner.prunePartitions(context, + scanOperator = OptExternalPartitionPruner.prunePartitions(optimizerContext, copiedScanOperator); } return OptExpression.create(scanOperator); } - public OptExpression visit(OptExpression optExpression, OptimizerContext context) { + public OptExpression visit(OptExpression optExpression, Void context) { List children = Lists.newArrayList(); for (int i = 0; i < optExpression.arity(); ++i) { - children.add(optExpression.inputAt(i).getOp().accept(this, optExpression.inputAt(i), context)); + children.add(optExpression.inputAt(i).getOp().accept(this, optExpression.inputAt(i), null)); } return OptExpression.create(optExpression.getOp(), children); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java index 45125b8d6c8de..6700d5bf14722 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java @@ -27,8 +27,11 @@ import com.google.common.graph.MutableGraph; import com.starrocks.analysis.JoinOperator; import com.starrocks.catalog.Column; +import com.starrocks.catalog.DistributionInfo; import com.starrocks.catalog.ForeignKeyConstraint; +import com.starrocks.catalog.HashDistributionInfo; import com.starrocks.catalog.KeysType; +import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; import com.starrocks.catalog.UniqueConstraint; @@ -155,22 +158,9 @@ public OptExpression rewrite() { final ColumnRefFactory mvColumnRefFactory = materializationContext.getMvColumnRefFactory(); final ReplaceColumnRefRewriter mvColumnRefRewriter = MvUtils.getReplaceColumnRefWriter(mvExpression, mvColumnRefFactory); - // Compensate partition predicates and add them into mv predicate, - // eg: c3 is partition column - // MV : select c1, c3, c2 from test_base_part where c3 < 2000 - // Query : select c1, c3, c2 from test_base_part - // `c3 < 2000` is missed after partition pruning, so `mvPredicate` must add `mvPartitionPredicate`, - // otherwise query above may be rewritten by mv. - final ScalarOperator mvPartitionPredicate = - MvUtils.compensatePartitionPredicate(mvExpression, mvColumnRefFactory); - if (mvPartitionPredicate == null) { - return null; - } + ScalarOperator mvPredicate = MvUtils.rewriteOptExprCompoundPredicate(mvExpression, mvColumnRefRewriter); - if (!ConstantOperator.TRUE.equals(mvPartitionPredicate)) { - mvPredicate = MvUtils.canonizePredicate(Utils.compoundAnd(mvPredicate, mvPartitionPredicate)); - } if (materializationContext.getMvPartialPartitionPredicate() != null) { // add latest partition predicate to mv predicate ScalarOperator rewritten = mvColumnRefRewriter.rewrite(materializationContext.getMvPartialPartitionPredicate()); @@ -222,7 +212,12 @@ public OptExpression rewrite() { materializationContext.getMvColumnRefFactory(), mvColumnRefRewriter, materializationContext.getOutputMapping(), queryColumnSet); + // collect partition and distribution related predicates in mv + // used to prune partition and buckets after mv rewrite + ScalarOperator mvPrunePredicate = collectMvPrunePredicate(materializationContext); + for (BiMap relationIdMapping : relationIdMappings) { + mvRewriteContext.setMvPruneConjunct(mvPrunePredicate); rewriteContext.setQueryToMvRelationIdMapping(relationIdMapping); // for view delta, should add compensation join columns to query ec @@ -502,6 +497,45 @@ private boolean isJoinMatch(OptExpression queryExpression, } } + private ScalarOperator collectMvPrunePredicate(MaterializationContext mvContext) { + final OptExpression mvExpression = mvContext.getMvExpression(); + final List conjuncts = MvUtils.getAllPredicates(mvExpression); + final ColumnRefSet mvOutputColumnRefSet = mvExpression.getOutputColumns(); + // conjuncts related to partition and distribution + final List mvPrunePredicates = Lists.newArrayList(); + + // Construct partition/distribution key column refs to filter conjunctions which need to retain. + Set mvPruneKeyColNames = Sets.newHashSet(); + MaterializedView mv = mvContext.getMv(); + DistributionInfo distributionInfo = mv.getDefaultDistributionInfo(); + // only hash distribution is supported + Preconditions.checkState(distributionInfo instanceof HashDistributionInfo); + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + List distributedColumns = hashDistributionInfo.getDistributionColumns(); + distributedColumns.stream().forEach(distKey -> mvPruneKeyColNames.add(distKey.getName())); + mv.getPartitionColumnNames().stream().forEach(partName -> mvPruneKeyColNames.add(partName)); + final Set mvPruneColumnIdSet = mvOutputColumnRefSet.getStream().map( + id -> mvContext.getMvColumnRefFactory().getColumnRef(id)) + .filter(colRef -> mvPruneKeyColNames.contains(colRef.getName())) + .map(colRef -> colRef.getId()) + .collect(Collectors.toSet()); + for (ScalarOperator conj : conjuncts) { + // ignore binary predicates which cannot be used for pruning. + if (conj instanceof BinaryPredicateOperator) { + BinaryPredicateOperator conjOp = (BinaryPredicateOperator) conj; + if (conjOp.getChild(0).isVariable() && conjOp.getChild(1).isVariable()) { + continue; + } + } + final List conjColumnRefOperators = + Utils.extractColumnRef(conj).stream().map(ref -> ref.getId()).collect(Collectors.toList()); + if (mvPruneColumnIdSet.containsAll(conjColumnRefOperators)) { + mvPrunePredicates.add(conj); + } + } + return Utils.compoundAnd(mvPrunePredicates); + } + private OptExpression tryRewriteForRelationMapping(RewriteContext rewriteContext) { // the rewritten expression to replace query // should copy the op because the op will be modified and reused @@ -512,13 +546,11 @@ private OptExpression tryRewriteForRelationMapping(RewriteContext rewriteContext // Rewrite original mv's predicates into query if needed. final ColumnRewriter columnRewriter = new ColumnRewriter(rewriteContext); final Map mvColumnRefToScalarOp = rewriteContext.getMVColumnRefToScalarOp(); - ScalarOperator mvOriginalPredicates = mvScanOperator.getPredicate(); - if (mvOriginalPredicates != null && !ConstantOperator.TRUE.equals(mvOriginalPredicates)) { - mvOriginalPredicates = rewriteMVCompensationExpression(rewriteContext, columnRewriter, - mvColumnRefToScalarOp, mvOriginalPredicates, false); - if (!ConstantOperator.TRUE.equals(mvOriginalPredicates)) { - mvScanBuilder.setPredicate(mvOriginalPredicates); - } + if (mvRewriteContext.getMvPruneConjunct() != null + && !ConstantOperator.TRUE.equals(mvRewriteContext.getMvPruneConjunct())) { + ScalarOperator rewrittenPrunePredicate = rewriteMVCompensationExpression(rewriteContext, columnRewriter, + mvColumnRefToScalarOp, mvRewriteContext.getMvPruneConjunct(), false); + mvRewriteContext.setMvPruneConjunct(MvUtils.canonizePredicate(rewrittenPrunePredicate)); } OptExpression mvScanOptExpression = OptExpression.create(mvScanBuilder.build()); deriveLogicalProperty(mvScanOptExpression); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index 2bbc5c4930251..d2c2e163f6f19 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -320,6 +320,7 @@ public static Pair getRuleOptimizedLogicalPlan(Strin new RelationTransformer(columnRefFactory, connectContext).transformWithSelectLimit(query); // optimize the sql by rule and disable rule based materialized view rewrite OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); + optimizerConfig.disableRuleSet(RuleSetType.PARTITION_PRUNE); optimizerConfig.disableRuleSet(RuleSetType.SINGLE_TABLE_MV_REWRITE); Optimizer optimizer = new Optimizer(optimizerConfig); OptExpression optimizedPlan = optimizer.optimize( @@ -382,12 +383,14 @@ private static void getAllPredicates(OptExpression root, List pr // Ignore aggregation predicates, because aggregation predicates should be rewritten after // aggregation functions' rewrite and should not be pushed down into mv scan operator. if (operator.getPredicate() != null && !(operator instanceof LogicalAggregationOperator)) { - predicates.add(operator.getPredicate()); + List conjuncts = Utils.extractConjuncts(operator.getPredicate()); + predicates.addAll(conjuncts); } if (operator instanceof LogicalJoinOperator) { LogicalJoinOperator joinOperator = (LogicalJoinOperator) operator; if (joinOperator.getOnPredicate() != null) { - predicates.add(joinOperator.getOnPredicate()); + List conjuncts = Utils.extractConjuncts(joinOperator.getOnPredicate()); + predicates.addAll(conjuncts); } } for (OptExpression child : root.getInputs()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java index 5d3a13b7cbb3d..244168a5c3996 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/BaseMaterializedViewRewriteRule.java @@ -88,7 +88,7 @@ public List transform(OptExpression queryExpression, OptimizerCon MaterializedViewRewriter mvRewriter = getMaterializedViewRewrite(mvRewriteContext); OptExpression candidate = mvRewriter.rewrite(); if (candidate != null) { - candidate = postRewriteMV(context, candidate); + candidate = postRewriteMV(context, mvRewriteContext, candidate); if (queryExpression.getGroupExpression() != null) { int currentRootGroupId = queryExpression.getGroupExpression().getGroup().getId(); mvContext.addMatchedGroup(currentRootGroupId); @@ -106,12 +106,13 @@ public List transform(OptExpression queryExpression, OptimizerCon * 2. partition prune * 3. bucket prune */ - private OptExpression postRewriteMV(OptimizerContext context, OptExpression candidate) { + private OptExpression postRewriteMV( + OptimizerContext optimizerContext, MvRewriteContext mvRewriteContext, OptExpression candidate) { if (candidate == null) { return null; } candidate = new MVColumnPruner().pruneColumns(candidate); - candidate = new MVPartitionPruner().prunePartition(context, candidate); + candidate = new MVPartitionPruner(optimizerContext, mvRewriteContext).prunePartition(candidate); return candidate; } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java index 0245efb75932e..4e59dfb6b1506 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java @@ -1543,8 +1543,7 @@ public void testViewDeltaJoinUKFK5() { testRewriteOK(mv, query) .contains("0:OlapScanNode\n" + " TABLE: mv0\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: 7: empid = 1"); + " PREAGGREGATION: ON\n"); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java index bcb5c88a7ae5e..c12d7112b91a6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTestBase.java @@ -175,7 +175,12 @@ public MVRewriteChecker nonMatch() { } public MVRewriteChecker contains(String expect) { - Assert.assertTrue(this.rewritePlan.contains(expect)); + boolean contained = this.rewritePlan.contains(expect); + if (!contained) { + LOG.warn("rewritePlan: \n{}", rewritePlan); + LOG.warn("expect: \n{}", expect); + } + Assert.assertTrue(contained); return this; } diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java index 952de5274095b..aafd5d3f11251 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewWithPartitionTest.java @@ -14,13 +14,10 @@ package com.starrocks.planner; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.BeforeClass; import org.junit.Test; public class MaterializedViewWithPartitionTest extends MaterializedViewTestBase { - private static final Logger LOG = LogManager.getLogger(MaterializedViewWithPartitionTest.class); @BeforeClass public static void setUp() throws Exception { @@ -127,10 +124,8 @@ public void testPartitionPrune_SingleTable2() throws Exception { .contains("UNION") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 6: c3 < 2000\n" + " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2") + " rollup: partial_mv_6") .contains("TABLE: test_base_part\n" + " PREAGGREGATION: ON\n" + " partitions=1/5\n" + @@ -143,10 +138,7 @@ public void testPartitionPrune_SingleTable2() throws Exception { .contains("partial_mv_6") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 6: c3 <= 999, 6: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 6: c3 <= 999"); starRocksAssert.dropMaterializedView("partial_mv_6"); } @@ -338,10 +330,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 2000") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 11: c3 < 2000\n" + " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " rollup: partial_mv_6"); // test query delta testRewriteOK( @@ -351,10 +341,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 2000 and t2.c3 > 100;") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 11: c3 <= 1999, 11: c3 >= 101, 11: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 11: c3 <= 1999, 11: c3 >= 101\n" + + " partitions=1/1"); // test query delta testRewriteOK( @@ -364,10 +352,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 1000;") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 11: c3 <= 999, 11: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 11: c3 <= 999\n" + + " partitions=1/1"); // test query delta, agg + join testRewriteOK( @@ -377,10 +363,8 @@ public void testPartitionPrune_MultiTables2() throws Exception { " where t1.c3 < 1000;") .contains("TABLE: partial_mv_6\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 12: c3 <= 999, 12: c3 < 2000\n" + - " partitions=1/1\n" + - " rollup: partial_mv_6\n" + - " tabletRatio=2/2"); + " PREDICATES: 12: c3 <= 999\n" + + " partitions=1/1"); // test union all // TODO: MV can be rewritten but cannot bingo(because cost?)! @@ -435,21 +419,15 @@ public void testBucketPrune_SingleTable1() throws Exception { .contains("UNION") .contains("TABLE: partial_mv_7\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 5: c1 = 1\n" + " partitions=4/5\n" + " rollup: partial_mv_7\n" + " tabletRatio=4/8") - .contains("TABLE: test_base_part\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: (8: c1 != 1) OR (10: c3 >= 2000)\n" + - " partitions=5/5\n" + - " rollup: test_base_part"); + .contains("TABLE: test_base_part"); // match all testRewriteOK("select c1, c3, c2 from test_base_part where c3 < 2000 and c1 = 1") .contains("TABLE: partial_mv_7\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 5: c1 = 1\n" + " partitions=4/5\n" + " rollup: partial_mv_7\n" + " tabletRatio=4/8"); @@ -458,7 +436,7 @@ public void testBucketPrune_SingleTable1() throws Exception { testRewriteOK("select c1, c3, c2 from test_base_part where c3 < 1000 and c1 = 1") .contains("TABLE: partial_mv_7\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 6: c3 <= 999, 5: c1 = 1\n" + + " PREDICATES: 6: c3 <= 999\n" + " partitions=3/5\n" + " rollup: partial_mv_7\n" + " tabletRatio=3/6"); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index c73fff5064f8f..4fa12e821b394 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -266,7 +266,6 @@ public void testSingleTableEqualPredicateRewrite() throws Exception { " 0:OlapScanNode\n" + " TABLE: mv_1\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 5: empid = 5\n" + " partitions=1/1\n" + " rollup: mv_1"); PlanTestBase.assertContains(plan, "tabletRatio=1/6"); @@ -293,7 +292,6 @@ public void testSingleTableEqualPredicateRewrite() throws Exception { " 0:OlapScanNode\n" + " TABLE: mv_1\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 7: empid = 5\n" + " partitions=1/1\n" + " rollup: mv_1\n" + " tabletRatio=1/6"); @@ -365,7 +363,6 @@ public void testSingleTableRangePredicateRewrite() throws Exception { " 0:OlapScanNode\n" + " TABLE: mv_1\n" + " PREAGGREGATION: ON\n" + - " PREDICATES: 5: empid < 5\n" + " partitions=1/1\n" + " rollup: mv_1\n" + " tabletRatio=6/6"); @@ -1738,11 +1735,7 @@ public void testPartialPartition() throws Exception { " select c1, c3, c2 from test_base_part where c3 < 2000 and c1 = 1;"); String query11 = "select c1, c3, c2 from test_base_part"; String plan11 = getFragmentPlan(query11); - PlanTestBase.assertContains(plan11, "partial_mv_7", "UNION", "TABLE: test_base_part\n" + - " PREAGGREGATION: ON\n" + - " PREDICATES: (8: c1 != 1) OR (10: c3 >= 2000)\n" + - " partitions=6/6\n" + - " rollup: test_base_part"); + PlanTestBase.assertContains(plan11, "partial_mv_7", "UNION", "TABLE: test_base_part"); dropMv("test", "partial_mv_7"); createAndRefreshMv("test", "partial_mv_8", "create materialized view partial_mv_8" +