Skip to content

Fix missing materialized column for hudi table #5119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dirtysalt merged 3 commits into from
Apr 18, 2022
Merged

Fix missing materialized column for hudi table #5119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
10d3161
Fix missing materialized column for hudi table
miomiocat Apr 15, 2022
52a63f8
refine
miomiocat Apr 15, 2022
3a079bf
ut
miomiocat Apr 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions ...e/src/main/java/com/starrocks/sql/optimizer/operator/logical/LogicalHiveScanOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
public class LogicalHiveScanOperator extends LogicalScanOperator {
private final Table.TableType tableType;
private ScanOperatorPredicates predicates = new ScanOperatorPredicates();
private Set<String> partitionColumns = Sets.newHashSet();

public LogicalHiveScanOperator(Table table,
Table.TableType tableType,
Expand Down Expand Up @@ -58,10 +57,6 @@ public Table.TableType getTableType() {
return tableType;
}

public Set<String> getPartitionColumns() {
return partitionColumns;
}

@Override
public ScanOperatorPredicates getScanOperatorPredicates() {
return this.predicates;
Expand Down
7 changes: 2 additions & 5 deletions ...e/src/main/java/com/starrocks/sql/optimizer/operator/logical/LogicalHudiScanOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
public class LogicalHudiScanOperator extends LogicalScanOperator {
private final Table.TableType tableType;
private ScanOperatorPredicates predicates = new ScanOperatorPredicates();
private Set<String> partitionColumns = Sets.newHashSet();

public LogicalHudiScanOperator(Table table,
Table.TableType tableType,
Expand All @@ -36,6 +35,8 @@ public LogicalHudiScanOperator(Table table,

Preconditions.checkState(table instanceof HudiTable);
this.tableType = tableType;
HudiTable hudiTable = (HudiTable) table;
partitionColumns.addAll(hudiTable.getPartitionColumnNames());
}

private LogicalHudiScanOperator(LogicalHudiScanOperator.Builder builder) {
Expand All @@ -56,10 +57,6 @@ public Table.TableType getTableType() {
return tableType;
}

public Set<String> getPartitionColumns() {
return partitionColumns;
}

@Override
public ScanOperatorPredicates getScanOperatorPredicates() {
return this.predicates;
Expand Down
7 changes: 7 additions & 0 deletions ...-core/src/main/java/com/starrocks/sql/optimizer/operator/logical/LogicalScanOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
package com.starrocks.sql.optimizer.operator.logical;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.starrocks.catalog.Column;
import com.starrocks.catalog.Table;
import com.starrocks.common.AnalysisException;
Expand All @@ -23,6 +24,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

public abstract class LogicalScanOperator extends LogicalOperator {
protected final Table table;
Expand All @@ -34,6 +36,7 @@ public abstract class LogicalScanOperator extends LogicalOperator {
protected final ImmutableMap<ColumnRefOperator, Column> colRefToColumnMetaMap;
protected final ImmutableMap<Column, ColumnRefOperator> columnMetaToColRefMap;
protected final ImmutableMap<String, PartitionColumnFilter> columnFilters;
protected Set<String> partitionColumns = Sets.newHashSet();

public LogicalScanOperator(
OperatorType type,
Expand Down Expand Up @@ -79,6 +82,10 @@ public List<ColumnRefOperator> getOutputColumns() {
return new ArrayList<>(colRefToColumnMetaMap.keySet());
}

public Set<String> getPartitionColumns() {
return partitionColumns;
}

@Override
public ColumnRefSet getOutputColumns(ExpressionContext expressionContext) {
if (projection != null) {
Expand Down
18 changes: 4 additions & 14 deletions ...rc/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,22 +134,12 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
}

private boolean containsMaterializedColumn(LogicalScanOperator scanOperator, Set<ColumnRefOperator> scanColumns) {
if (scanOperator instanceof LogicalHiveScanOperator) {
return scanColumns.size() != 0 && !((LogicalHiveScanOperator) scanOperator).getPartitionColumns().containsAll(
scanColumns.stream().map(ColumnRefOperator::getName).collect(Collectors.toList()));
}
return scanColumns.size() != 0;
return scanColumns.size() != 0 && !scanOperator.getPartitionColumns().containsAll(
scanColumns.stream().map(ColumnRefOperator::getName).collect(Collectors.toList()));
}

private boolean isPartitionColumn(LogicalScanOperator scanOperator, String columnName) {
if (scanOperator instanceof LogicalHiveScanOperator) {
// Hive partition columns is not materialized column, so except partition columns
return ((LogicalHiveScanOperator) scanOperator).getPartitionColumns().contains(columnName);
}
if (scanOperator instanceof LogicalHudiScanOperator) {
// Hudi partition columns is not materialized column, so except partition columns
return ((LogicalHudiScanOperator) scanOperator).getPartitionColumns().contains(columnName);
}
return false;
// Hive/Hudi partition columns is not materialized column, so except partition columns
return scanOperator.getPartitionColumns().contains(columnName);
}
}
68 changes: 64 additions & 4 deletions ...est/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@

import com.google.common.collect.Maps;
import com.starrocks.catalog.Column;
import com.starrocks.catalog.HudiTable;
import com.starrocks.catalog.IcebergTable;
import com.starrocks.catalog.Table;
import com.starrocks.catalog.Type;
import com.starrocks.sql.optimizer.OptExpression;
import com.starrocks.sql.optimizer.OptimizerContext;
import com.starrocks.sql.optimizer.base.ColumnRefSet;
import com.starrocks.sql.optimizer.operator.logical.LogicalHudiScanOperator;
import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator;
import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
Expand All @@ -28,6 +30,7 @@

public class PruneHDFSScanColumnRuleTest {
private PruneHDFSScanColumnRule icebergRule = PruneHDFSScanColumnRule.ICEBERG_SCAN;
private PruneHDFSScanColumnRule hudiRule = PruneHDFSScanColumnRule.HUDI_SCAN;

ColumnRefOperator intColumnOperator = new ColumnRefOperator(1, Type.INT, "id", true);
ColumnRefOperator strColumnOperator = new ColumnRefOperator(2, Type.STRING, "name", true);
Expand All @@ -39,8 +42,8 @@ public class PruneHDFSScanColumnRuleTest {

@Test
public void transformIcebergWithPredicate(@Mocked IcebergTable table,
@Mocked OptimizerContext context,
@Mocked TaskContext taskContext) {
@Mocked OptimizerContext context,
@Mocked TaskContext taskContext) {
OptExpression scan = new OptExpression(
new LogicalIcebergScanOperator(table, Table.TableType.ICEBERG,
scanColumnMap, Maps.newHashMap(), -1,
Expand All @@ -59,8 +62,8 @@ public void transformIcebergWithPredicate(@Mocked IcebergTable table,

@Test
public void transformIcebergWithNoScanColumn(@Mocked IcebergTable table,
@Mocked OptimizerContext context,
@Mocked TaskContext taskContext) {
@Mocked OptimizerContext context,
@Mocked TaskContext taskContext) {
OptExpression scan = new OptExpression(
new LogicalIcebergScanOperator(table, Table.TableType.ICEBERG,
scanColumnMap, Maps.newHashMap(), -1, null));
Expand Down Expand Up @@ -93,4 +96,61 @@ private void doIcebergTransform(OptExpression scan,
Assert.assertEquals(transferMap.size(), 1);
Assert.assertEquals(transferMap.get(intColumnOperator).getName(), "id");
}

@Test
public void transformHudiWithPredicate(@Mocked HudiTable table,
@Mocked OptimizerContext context,
@Mocked TaskContext taskContext) {
OptExpression scan = new OptExpression(
new LogicalHudiScanOperator(table, Table.TableType.HUDI,
scanColumnMap, Maps.newHashMap(), -1,
new BinaryPredicateOperator(BinaryPredicateOperator.BinaryType.EQ,
new ColumnRefOperator(1, Type.INT, "id", true),
ConstantOperator.createInt(1))));

List<TaskContext> taskContextList = new ArrayList<>();
taskContextList.add(taskContext);

ColumnRefSet requiredOutputColumns = new ColumnRefSet(new ArrayList<>(
Collections.singleton(new ColumnRefOperator(1, Type.INT, "id", true))));

doHudiTransform(scan, context, requiredOutputColumns, taskContextList, taskContext);
}

@Test
public void transformHudiWithNoScanColumn(@Mocked HudiTable table,
@Mocked OptimizerContext context,
@Mocked TaskContext taskContext) {
OptExpression scan = new OptExpression(
new LogicalHudiScanOperator(table, Table.TableType.HUDI,
scanColumnMap, Maps.newHashMap(), -1, null));

List<TaskContext> taskContextList = new ArrayList<>();
taskContextList.add(taskContext);

ColumnRefSet requiredOutputColumns = new ColumnRefSet(new ArrayList<>());

doHudiTransform(scan, context, requiredOutputColumns, taskContextList, taskContext);
}

private void doHudiTransform(OptExpression scan,
OptimizerContext context,
ColumnRefSet requiredOutputColumns,
List<TaskContext> taskContextList,
TaskContext taskContext) {
new Expectations() { {
context.getTaskContext();
minTimes = 0;
result = taskContextList;

taskContext.getRequiredColumns();
minTimes = 0;
result = requiredOutputColumns;
}};
List<OptExpression> list = hudiRule.transform(scan, context);
Map<ColumnRefOperator, Column> transferMap = ((LogicalHudiScanOperator)list.get(0)
.getOp()).getColRefToColumnMetaMap();
Assert.assertEquals(transferMap.size(), 1);
Assert.assertEquals(transferMap.get(intColumnOperator).getName(), "id");
}
}