diff --git a/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java b/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java index 84c71f1111688..46ed546dcd607 100644 --- a/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java +++ b/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.assertions.BasePlanTest; @@ -36,9 +37,9 @@ import java.nio.file.Paths; import java.util.stream.Stream; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.testing.TestngUtils.toDataProvider; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; @@ -157,7 +158,7 @@ public String result() @Override public Void visitJoin(JoinNode node, Integer indent) { - JoinNode.DistributionType distributionType = node.getDistributionType() + JoinDistributionType distributionType = node.getDistributionType() .orElseThrow(() -> new VerifyException("Expected distribution type to be set")); if (node.isCrossJoin()) { checkState(node.getType() == INNER && distributionType == REPLICATED, "Expected CROSS JOIN to be INNER REPLICATED"); diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index c5952654d5b8f..9de464f3e90a8 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -216,6 +216,9 @@ Property Name Description ``iceberg.enable-merge-on-read-mode`` Enable reading base tables that use merge-on-read for ``true`` updates. + +``iceberg.delete-as-join-rewrite-enabled`` When enabled, equality delete row filtering is applied ``true`` + as a join with the data of the equality delete files. ================================================== ============================================================= ============ Table Properties @@ -269,6 +272,18 @@ and a file system location of ``s3://test_bucket/test_schema/test_table``: location = 's3://test_bucket/test_schema/test_table') ) +Session Properties +------------------- + +Session properties set behavior changes for queries executed within the given session. + +============================================= ====================================================================== +Property Name Description +============================================= ====================================================================== +``iceberg.delete_as_join_rewrite_enabled`` Overrides the behavior of the connector property + ``iceberg.delete-as-join-rewrite-enabled`` in the current session. +============================================= ====================================================================== + Caching Support ---------------- @@ -373,6 +388,38 @@ Metastore cache only caches schema and table names. Other metadata would be fetc hive.metastore-refresh-interval=3d hive.metastore-cache-maximum-size=10000000 +Extra Hidden Metadata Columns +---------------------------- + +The Iceberg connector exposes extra hidden metadata columns. You can query these +as part of a SQL query by including them in your SELECT statement. + +``$path`` column +^^^^^^^^^^^^^^^^ +* ``$path``: Full file system path name of the file for this row +.. code-block:: sql + + SELECT "$path", regionkey FROM "ctas_nation"; + +.. code-block:: text + + $path | regionkey + ---------------------------------+----------- + /full/path/to/file/file.parquet | 2 + +``$data_sequence_number`` column +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* ``$data_sequence_number``: The Iceberg data sequence number in which this row was added +.. code-block:: sql + + SELECT "$data_sequence_number", regionkey FROM "ctas_nation"; + +.. code-block:: text + + $data_sequence_number | regionkey + ----------------------------------+------------ + 2 | 3 + Extra Hidden Metadata Tables ---------------------------- diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java index 5c25c2bca80c0..ffc7876a40531 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java @@ -17,6 +17,7 @@ import com.facebook.presto.execution.SqlQueryManager; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.sql.planner.Plan; @@ -131,7 +132,7 @@ public void testBroadcastJoin() "(SELECT * FROM test_orders where ds = '2020-09-02' and substr(CAST(custkey AS VARCHAR), 1, 3) = '370') t2 ON t1.orderkey = t2.orderkey", defaultSession()); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) - .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinNode.DistributionType.PARTITIONED)) + .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinDistributionType.PARTITIONED)) .findFirst() .isPresent()); @@ -146,7 +147,7 @@ public void testBroadcastJoin() "(SELECT * FROM test_orders where ds = '2020-09-02' and substr(CAST(custkey AS VARCHAR), 1, 3) = '370') t2 ON t1.orderkey = t2.orderkey", defaultSession()); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) - .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinNode.DistributionType.REPLICATED)) + .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinDistributionType.REPLICATED)) .findFirst() .isPresent()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java index e5bf1eb895352..f42aa4c1de014 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java @@ -111,6 +111,7 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; import static com.facebook.presto.parquet.ParquetTypeUtils.pushdownColumnNameForSubfield; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.assertions.MatchResult.NO_MATCH; import static com.facebook.presto.sql.planner.assertions.MatchResult.match; @@ -133,7 +134,6 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.collect.ImmutableList.toImmutableList; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java index a4a82fe0a758f..8f1dbd7d3c2e8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java @@ -56,6 +56,8 @@ import static com.facebook.presto.hive.TestHiveLogicalPlanner.replicateHiveMetastore; import static com.facebook.presto.hive.TestHiveLogicalPlanner.utf8Slices; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -71,8 +73,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.unnest; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; import static com.facebook.presto.testing.TestingAccessControlManager.TestingPrivilegeType.INSERT_TABLE; import static com.facebook.presto.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN; import static com.facebook.presto.testing.TestingAccessControlManager.privilege; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestMergeJoinPlan.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestMergeJoinPlan.java index a7336feb82f8b..dc1a093fe1cdd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestMergeJoinPlan.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestMergeJoinPlan.java @@ -14,8 +14,8 @@ package com.facebook.presto.hive; import com.facebook.presto.Session; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; -import com.facebook.presto.sql.planner.plan.JoinNode.Type; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableList; @@ -29,15 +29,15 @@ import static com.facebook.presto.SystemSessionProperties.PREFER_MERGE_JOIN_FOR_SORTED_INPUTS; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveSessionProperties.ORDER_BASED_EXECUTION_ENABLED; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.mergeJoin; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static io.airlift.tpch.TpchTable.CUSTOMER; import static io.airlift.tpch.TpchTable.LINE_ITEM; import static io.airlift.tpch.TpchTable.NATION; @@ -316,7 +316,7 @@ private Session mergeJoinEnabled() .build(); } - private PlanMatchPattern joinPlan(String leftTableName, String rightTableName, List leftJoinKeys, List rightJoinKeys, Type joinType, boolean mergeJoinEnabled) + private PlanMatchPattern joinPlan(String leftTableName, String rightTableName, List leftJoinKeys, List rightJoinKeys, JoinType joinType, boolean mergeJoinEnabled) { int suffix1 = 0; int suffix2 = 1; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ColumnIdentity.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ColumnIdentity.java index c03db26ad47dd..035cb560eb740 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ColumnIdentity.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/ColumnIdentity.java @@ -121,10 +121,11 @@ public static ColumnIdentity primitiveColumnIdentity(int id, String name) public static ColumnIdentity createColumnIdentity(Types.NestedField column) { - int id = column.fieldId(); - String name = column.name(); - org.apache.iceberg.types.Type fieldType = column.type(); + return createColumnIdentity(column.name(), column.fieldId(), column.type()); + } + public static ColumnIdentity createColumnIdentity(String name, int id, org.apache.iceberg.types.Type fieldType) + { if (fieldType.equals(Types.TimestampType.withZone())) { throw new UnsupportedOperationException(format("Iceberg column type %s is not supported", fieldType)); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java index fde213fa6eee3..8198a46a58a70 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java @@ -103,8 +103,14 @@ import static com.facebook.presto.hive.MetadataUtils.getPredicate; import static com.facebook.presto.hive.MetadataUtils.getSubfieldPredicate; import static com.facebook.presto.iceberg.ExpressionConverter.toIcebergExpression; +import static com.facebook.presto.iceberg.IcebergColumnHandle.DATA_SEQUENCE_NUMBER_COLUMN_HANDLE; +import static com.facebook.presto.iceberg.IcebergColumnHandle.DATA_SEQUENCE_NUMBER_COLUMN_METADATA; +import static com.facebook.presto.iceberg.IcebergColumnHandle.PATH_COLUMN_HANDLE; +import static com.facebook.presto.iceberg.IcebergColumnHandle.PATH_COLUMN_METADATA; import static com.facebook.presto.iceberg.IcebergColumnHandle.primitiveIcebergColumnHandle; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_SNAPSHOT_ID; +import static com.facebook.presto.iceberg.IcebergMetadataColumn.DATA_SEQUENCE_NUMBER; +import static com.facebook.presto.iceberg.IcebergMetadataColumn.FILE_PATH; import static com.facebook.presto.iceberg.IcebergSessionProperties.isPushdownFilterEnabled; import static com.facebook.presto.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static com.facebook.presto.iceberg.IcebergTableProperties.FORMAT_VERSION; @@ -112,6 +118,7 @@ import static com.facebook.presto.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static com.facebook.presto.iceberg.IcebergTableType.CHANGELOG; import static com.facebook.presto.iceberg.IcebergTableType.DATA; +import static com.facebook.presto.iceberg.IcebergTableType.EQUALITY_DELETES; import static com.facebook.presto.iceberg.IcebergUtil.getColumns; import static com.facebook.presto.iceberg.IcebergUtil.getFileFormat; import static com.facebook.presto.iceberg.IcebergUtil.getPartitionKeyColumnHandles; @@ -138,7 +145,6 @@ import static java.lang.String.format; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; -import static java.util.function.Function.identity; public abstract class IcebergAbstractMetadata implements ConnectorMetadata @@ -334,11 +340,16 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect protected ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName table, IcebergTableName icebergTableName) { Table icebergTable = getIcebergTable(session, new SchemaTableName(table.getSchemaName(), icebergTableName.getTableName())); - List columns = getColumnMetadatas(icebergTable); + ImmutableList.Builder columns = ImmutableList.builder(); + columns.addAll(getColumnMetadatas(icebergTable)); if (icebergTableName.getTableType() == CHANGELOG) { - return ChangelogUtil.getChangelogTableMeta(table, typeManager, columns); + return ChangelogUtil.getChangelogTableMeta(table, typeManager, columns.build()); } - return new ConnectorTableMetadata(table, columns, createMetadataProperties(icebergTable), getTableComment(icebergTable)); + else { + columns.add(PATH_COLUMN_METADATA); + columns.add(DATA_SEQUENCE_NUMBER_COLUMN_METADATA); + } + return new ConnectorTableMetadata(table, columns.build(), createMetadataProperties(icebergTable), getTableComment(icebergTable)); } @Override @@ -632,8 +643,16 @@ public Map getColumnHandles(ConnectorSession session, Conn else { schema = icebergTable.schema(); } - return getColumns(schema, icebergTable.spec(), typeManager).stream() - .collect(toImmutableMap(IcebergColumnHandle::getName, identity())); + + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + for (IcebergColumnHandle columnHandle : getColumns(schema, icebergTable.spec(), typeManager)) { + columnHandles.put(columnHandle.getName(), columnHandle); + } + if (table.getIcebergTableName().getTableType() != CHANGELOG) { + columnHandles.put(FILE_PATH.getColumnName(), PATH_COLUMN_HANDLE); + columnHandles.put(DATA_SEQUENCE_NUMBER.getColumnName(), DATA_SEQUENCE_NUMBER_COLUMN_HANDLE); + } + return columnHandles.build(); } @Override @@ -654,7 +673,7 @@ public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableNa public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional tableVersion) { IcebergTableName name = IcebergTableName.from(tableName.getTableName()); - verify(name.getTableType() == DATA || name.getTableType() == CHANGELOG, "Wrong table type: " + name.getTableType()); + verify(name.getTableType() == DATA || name.getTableType() == CHANGELOG || name.getTableType() == EQUALITY_DELETES, "Wrong table type: " + name.getTableType()); if (!tableExists(session, tableName)) { return null; @@ -680,7 +699,9 @@ public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableNa new IcebergTableName(name.getTableName(), name.getTableType(), tableSnapshotId, name.getChangelogEndSnapshot()), name.getSnapshotId().isPresent(), TupleDomain.all(), - tableSchemaJson); + tableSchemaJson, + Optional.empty(), + Optional.empty()); } @Override diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergColumnHandle.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergColumnHandle.java index 14a410db75e38..9c74f8723cdf1 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergColumnHandle.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergColumnHandle.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.hive.BaseHiveColumnHandle; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -32,6 +33,8 @@ import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.SYNTHESIZED; import static com.facebook.presto.iceberg.ColumnIdentity.createColumnIdentity; import static com.facebook.presto.iceberg.ColumnIdentity.primitiveColumnIdentity; +import static com.facebook.presto.iceberg.IcebergMetadataColumn.DATA_SEQUENCE_NUMBER; +import static com.facebook.presto.iceberg.IcebergMetadataColumn.FILE_PATH; import static com.facebook.presto.iceberg.TypeConverter.toPrestoType; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Iterables.getOnlyElement; @@ -42,6 +45,11 @@ public class IcebergColumnHandle extends BaseHiveColumnHandle { + public static final IcebergColumnHandle PATH_COLUMN_HANDLE = getIcebergColumnHandle(FILE_PATH); + public static final ColumnMetadata PATH_COLUMN_METADATA = getColumnMetadata(FILE_PATH); + public static final IcebergColumnHandle DATA_SEQUENCE_NUMBER_COLUMN_HANDLE = getIcebergColumnHandle(DATA_SEQUENCE_NUMBER); + public static final ColumnMetadata DATA_SEQUENCE_NUMBER_COLUMN_METADATA = getColumnMetadata(DATA_SEQUENCE_NUMBER); + private final ColumnIdentity columnIdentity; private final Type type; @@ -132,6 +140,39 @@ public String toString() return getId() + ":" + getName() + ":" + type.getDisplayName() + ":" + getColumnType() + ":" + getRequiredSubfields(); } + private static IcebergColumnHandle getIcebergColumnHandle(IcebergMetadataColumn metadataColumn) + { + return new IcebergColumnHandle( + columIdentity(metadataColumn), + metadataColumn.getType(), + Optional.empty(), + SYNTHESIZED); + } + + private static ColumnMetadata getColumnMetadata(IcebergMetadataColumn metadataColumn) + { + return ColumnMetadata.builder() + .setName(metadataColumn.getColumnName()) + .setType(metadataColumn.getType()) + .setHidden(true) + .build(); + } + + private static ColumnIdentity columIdentity(IcebergMetadataColumn metadata) + { + return new ColumnIdentity(metadata.getId(), metadata.getColumnName(), metadata.getTypeCategory(), ImmutableList.of()); + } + + public boolean isPathColumn() + { + return getColumnIdentity().getId() == FILE_PATH.getId(); + } + + public boolean isDataSequenceNumberColumn() + { + return getColumnIdentity().getId() == DATA_SEQUENCE_NUMBER.getId(); + } + public static IcebergColumnHandle primitiveIcebergColumnHandle(int id, String name, Type type, Optional comment) { return new IcebergColumnHandle(primitiveColumnIdentity(id, name), type, comment, REGULAR); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java index 4dca155eef238..6f8d88f044739 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java @@ -50,6 +50,7 @@ public class IcebergConfig private boolean mergeOnReadModeEnabled = true; private double statisticSnapshotRecordDifferenceWeight; private boolean pushdownFilterEnabled; + private boolean deleteAsJoinRewriteEnabled = true; private HiveStatisticsMergeStrategy hiveStatisticsMergeStrategy = HiveStatisticsMergeStrategy.NONE; private String fileIOImpl = HadoopFileIO.class.getName(); @@ -235,6 +236,19 @@ public boolean isPushdownFilterEnabled() return pushdownFilterEnabled; } + @Config("iceberg.delete-as-join-rewrite-enabled") + @ConfigDescription("When enabled, equality delete row filtering will be implemented by rewriting the query plan to join with the delete keys.") + public IcebergConfig setDeleteAsJoinRewriteEnabled(boolean deleteAsJoinPushdownEnabled) + { + this.deleteAsJoinRewriteEnabled = deleteAsJoinPushdownEnabled; + return this; + } + + public boolean isDeleteAsJoinRewriteEnabled() + { + return deleteAsJoinRewriteEnabled; + } + public boolean getManifestCachingEnabled() { return manifestCachingEnabled; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataColumn.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataColumn.java new file mode 100644 index 0000000000000..a3e3d3e258cb1 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataColumn.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.common.type.Type; +import org.apache.iceberg.MetadataColumns; + +import java.util.Set; +import java.util.stream.Stream; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.iceberg.ColumnIdentity.TypeCategory.PRIMITIVE; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +public enum IcebergMetadataColumn +{ + FILE_PATH(MetadataColumns.FILE_PATH.fieldId(), "$path", VARCHAR, PRIMITIVE), + DATA_SEQUENCE_NUMBER(Integer.MAX_VALUE - 1001, "$data_sequence_number", BIGINT, PRIMITIVE), + /**/; + + private static final Set COLUMN_IDS = Stream.of(values()) + .map(IcebergMetadataColumn::getId) + .collect(toImmutableSet()); + private final int id; + private final String columnName; + private final Type type; + private final ColumnIdentity.TypeCategory typeCategory; + + IcebergMetadataColumn(int id, String columnName, Type type, ColumnIdentity.TypeCategory typeCategory) + { + this.id = id; + this.columnName = columnName; + this.type = type; + this.typeCategory = typeCategory; + } + + public int getId() + { + return id; + } + + public String getColumnName() + { + return columnName; + } + + public Type getType() + { + return type; + } + + public ColumnIdentity.TypeCategory getTypeCategory() + { + return typeCategory; + } + + public static boolean isMetadataColumnId(int id) + { + return COLUMN_IDS.contains(id); + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSource.java index 2567fda2d1927..ea6b958de3fa0 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSource.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSource.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.function.Supplier; +import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_BAD_DATA; import static com.facebook.presto.iceberg.IcebergUtil.deserializePartitionValue; import static com.google.common.base.Throwables.throwIfInstanceOf; @@ -49,6 +50,7 @@ public class IcebergPageSource public IcebergPageSource( List columns, + Map metadataValues, Map partitionKeys, ConnectorPageSource delegate, Supplier> deletePredicate) @@ -72,6 +74,16 @@ public IcebergPageSource( prefilledBlocks[outputIndex] = nativeValueToBlock(type, prefilledValue); delegateIndexes[outputIndex] = -1; } + else if (column.getColumnType() == PARTITION_KEY) { + // Partition key with no value. This can happen after partition evolution + Type type = column.getType(); + prefilledBlocks[outputIndex] = nativeValueToBlock(type, null); + delegateIndexes[outputIndex] = -1; + } + else if (IcebergMetadataColumn.isMetadataColumnId(column.getId())) { + prefilledBlocks[outputIndex] = nativeValueToBlock(column.getType(), metadataValues.get(column.getColumnIdentity().getId())); + delegateIndexes[outputIndex] = -1; + } else { delegateIndexes[outputIndex] = delegateIndex; delegateIndex++; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java index 84c46b5c2d18e..faa417dd0e13b 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java @@ -107,6 +107,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -118,6 +119,7 @@ import java.util.stream.IntStream; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.SYNTHESIZED; import static com.facebook.presto.hive.CacheQuota.NO_CACHE_CONSTRAINTS; @@ -724,13 +726,17 @@ public ConnectorPageSource createPageSource( List regularColumns = columns.stream() .map(IcebergColumnHandle.class::cast) - .filter(column -> !partitionKeys.containsKey(column.getId())) + .filter(column -> column.getColumnType() != PARTITION_KEY && + !partitionKeys.containsKey(column.getId()) && + !IcebergMetadataColumn.isMetadataColumnId(column.getId())) .collect(Collectors.toList()); Optional tableSchemaJson = table.getTableSchemaJson(); verify(tableSchemaJson.isPresent(), "tableSchemaJson is null"); Schema tableSchema = SchemaParser.fromJson(tableSchemaJson.get()); - Set deleteFilterRequiredColumns = requiredColumnsForDeletes(tableSchema, split.getDeletes()); + + boolean equalityDeletesRequired = table.getIcebergTableName().getTableType() == IcebergTableType.DATA; + Set deleteFilterRequiredColumns = requiredColumnsForDeletes(tableSchema, split.getDeletes(), equalityDeletesRequired); deleteFilterRequiredColumns.stream() .filter(not(icebergColumns::contains)) @@ -751,11 +757,17 @@ public ConnectorPageSource createPageSource( ConnectorPageSource dataPageSource = connectorPageSourceWithRowPositions.getConnectorPageSource(); Supplier> deletePredicate = Suppliers.memoize(() -> { + // If equality deletes are optimized into a join they don't need to be applied here + List deletesToApply = split + .getDeletes() + .stream() + .filter(deleteFile -> deleteFile.content() == POSITION_DELETES || equalityDeletesRequired) + .collect(toImmutableList()); List deleteFilters = readDeletes( session, tableSchema, split.getPath(), - split.getDeletes(), + deletesToApply, connectorPageSourceWithRowPositions.getStartRowPosition(), connectorPageSourceWithRowPositions.getEndRowPosition()); return deleteFilters.stream() @@ -763,21 +775,31 @@ public ConnectorPageSource createPageSource( .reduce(RowPredicate::and); }); - ConnectorPageSource dataSource = new IcebergPageSource(icebergColumns, partitionKeys, dataPageSource, deletePredicate); + HashMap metadataValues = new HashMap<>(); + for (IcebergColumnHandle icebergColumn : icebergColumns) { + if (icebergColumn.isPathColumn()) { + metadataValues.put(icebergColumn.getColumnIdentity().getId(), utf8Slice(split.getPath())); + } + else if (icebergColumn.isDataSequenceNumberColumn()) { + metadataValues.put(icebergColumn.getColumnIdentity().getId(), split.getDataSequenceNumber()); + } + } + + ConnectorPageSource dataSource = new IcebergPageSource(icebergColumns, metadataValues, partitionKeys, dataPageSource, deletePredicate); if (split.getChangelogSplitInfo().isPresent()) { dataSource = new ChangelogPageSource(dataSource, split.getChangelogSplitInfo().get(), (List) (List) desiredColumns, icebergColumns); } return dataSource; } - private Set requiredColumnsForDeletes(Schema schema, List deletes) + private Set requiredColumnsForDeletes(Schema schema, List deletes, boolean equalityDeletesRequired) { ImmutableSet.Builder requiredColumns = ImmutableSet.builder(); for (DeleteFile deleteFile : deletes) { if (deleteFile.content() == POSITION_DELETES) { requiredColumns.add(IcebergColumnHandle.create(ROW_POSITION, typeManager, IcebergColumnHandle.ColumnType.REGULAR)); } - else if (deleteFile.content() == EQUALITY_DELETES) { + else if (deleteFile.content() == EQUALITY_DELETES && equalityDeletesRequired) { deleteFile.equalityFieldIds().stream() .map(id -> IcebergColumnHandle.create(schema.findField(id), typeManager, IcebergColumnHandle.ColumnType.REGULAR)) .forEach(requiredColumns::add); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java index 5531efd425336..fe4c51aa75492 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java @@ -55,6 +55,7 @@ public final class IcebergSessionProperties public static final String PARQUET_DEREFERENCE_PUSHDOWN_ENABLED = "parquet_dereference_pushdown_enabled"; public static final String MERGE_ON_READ_MODE_ENABLED = "merge_on_read_enabled"; public static final String PUSHDOWN_FILTER_ENABLED = "pushdown_filter_enabled"; + public static final String DELETE_AS_JOIN_REWRITE_ENABLED = "delete_as_join_rewrite_enabled"; public static final String HIVE_METASTORE_STATISTICS_MERGE_STRATEGY = "hive_statistics_merge_strategy"; public static final String STATISTIC_SNAPSHOT_RECORD_DIFFERENCE_WEIGHT = "statistic_snapshot_record_difference_weight"; @@ -175,6 +176,11 @@ public IcebergSessionProperties( "value of 1 means a single record is equivalent to 1 millisecond of " + "time difference.", icebergConfig.getStatisticSnapshotRecordDifferenceWeight(), + false), + booleanProperty( + DELETE_AS_JOIN_REWRITE_ENABLED, + "When enabled equality delete row filtering will be pushed down into a join.", + icebergConfig.isDeleteAsJoinRewriteEnabled(), false)); } @@ -280,4 +286,9 @@ public static double getStatisticSnapshotRecordDifferenceWeight(ConnectorSession { return session.getProperty(STATISTIC_SNAPSHOT_RECORD_DIFFERENCE_WEIGHT, Double.class); } + + public static boolean isDeleteToJoinPushdownEnabled(ConnectorSession session) + { + return session.getProperty(DELETE_AS_JOIN_REWRITE_ENABLED, Boolean.class); + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplit.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplit.java index 013b9fe3c9927..bf807512272ab 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplit.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplit.java @@ -49,6 +49,7 @@ public class IcebergSplit private final SplitWeight splitWeight; private final List deletes; private final Optional changelogSplitInfo; + private final long dataSequenceNumber; @JsonCreator public IcebergSplit( @@ -61,7 +62,8 @@ public IcebergSplit( @JsonProperty("nodeSelectionStrategy") NodeSelectionStrategy nodeSelectionStrategy, @JsonProperty("splitWeight") SplitWeight splitWeight, @JsonProperty("deletes") List deletes, - @JsonProperty("changelogSplitInfo") Optional changelogSplitInfo) + @JsonProperty("changelogSplitInfo") Optional changelogSplitInfo, + @JsonProperty("dataSequenceNumber") long dataSequenceNumber) { requireNonNull(nodeSelectionStrategy, "nodeSelectionStrategy is null"); this.path = requireNonNull(path, "path is null"); @@ -74,6 +76,7 @@ public IcebergSplit( this.splitWeight = requireNonNull(splitWeight, "splitWeight is null"); this.deletes = ImmutableList.copyOf(requireNonNull(deletes, "deletes is null")); this.changelogSplitInfo = requireNonNull(changelogSplitInfo, "changelogSplitInfo is null"); + this.dataSequenceNumber = dataSequenceNumber; } @JsonProperty @@ -147,6 +150,12 @@ public Optional getChangelogSplitInfo() return changelogSplitInfo; } + @JsonProperty + public long getDataSequenceNumber() + { + return dataSequenceNumber; + } + @Override public Object getInfo() { diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java index b55196dc2ca8b..e9ed7f0454de3 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java @@ -16,6 +16,7 @@ import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.iceberg.changelog.ChangelogSplitSource; +import com.facebook.presto.iceberg.equalitydeletes.EqualityDeletesSplitSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.ConnectorTableLayoutHandle; @@ -23,9 +24,11 @@ import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.google.common.collect.ImmutableList; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.IncrementalChangelogScan; import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; +import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.util.SnapshotUtil; import org.apache.iceberg.util.TableScanUtil; @@ -36,6 +39,7 @@ import static com.facebook.presto.iceberg.IcebergSessionProperties.getMinimumAssignedSplitWeight; import static com.facebook.presto.iceberg.IcebergSessionProperties.isPushdownFilterEnabled; import static com.facebook.presto.iceberg.IcebergTableType.CHANGELOG; +import static com.facebook.presto.iceberg.IcebergTableType.EQUALITY_DELETES; import static com.facebook.presto.iceberg.IcebergUtil.getIcebergTable; import static java.util.Objects.requireNonNull; @@ -86,6 +90,15 @@ public ConnectorSplitSource getSplits( .toSnapshot(toSnapshot); return new ChangelogSplitSource(session, typeManager, icebergTable, scan, scan.targetSplitSize()); } + else if (table.getIcebergTableName().getTableType() == EQUALITY_DELETES) { + CloseableIterable deleteFiles = IcebergUtil.getDeleteFiles(icebergTable, + table.getIcebergTableName().getSnapshotId().get(), + table.getPredicate(), + table.getPartitionSpecId(), + table.getEqualityFieldIds()); + + return new EqualityDeletesSplitSource(session, icebergTable, deleteFiles); + } else { TableScan tableScan = icebergTable.newScan() .filter(toIcebergExpression(predicate)) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java index f901ec1f6bdd9..b908d9ef7c80a 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java @@ -35,6 +35,7 @@ import java.util.concurrent.CompletableFuture; import static com.facebook.presto.hive.HiveCommonSessionProperties.getNodeSelectionStrategy; +import static com.facebook.presto.iceberg.IcebergUtil.getDataSequenceNumber; import static com.facebook.presto.iceberg.IcebergUtil.getPartitionKeys; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterators.limit; @@ -114,6 +115,7 @@ private ConnectorSplit toIcebergSplit(FileScanTask task) getNodeSelectionStrategy(session), SplitWeight.fromProportion(Math.min(Math.max((double) task.length() / tableScan.targetSplitSize(), minimumAssignedSplitWeight), 1.0)), task.deletes().stream().map(DeleteFile::fromIceberg).collect(toImmutableList()), - Optional.empty()); + Optional.empty(), + getDataSequenceNumber(task.file())); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableHandle.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableHandle.java index e411e0ef72238..6c6be167e2faa 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableHandle.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableHandle.java @@ -20,6 +20,7 @@ import java.util.Objects; import java.util.Optional; +import java.util.Set; import static java.util.Objects.requireNonNull; @@ -30,6 +31,8 @@ public class IcebergTableHandle private final TupleDomain predicate; private final boolean snapshotSpecified; private final Optional tableSchemaJson; + private final Optional> partitionFieldIds; + private final Optional> equalityFieldIds; @JsonCreator public IcebergTableHandle( @@ -37,7 +40,9 @@ public IcebergTableHandle( @JsonProperty("icebergTableName") IcebergTableName icebergTableName, @JsonProperty("snapshotSpecified") boolean snapshotSpecified, @JsonProperty("predicate") TupleDomain predicate, - @JsonProperty("tableSchemaJson") Optional tableSchemaJson) + @JsonProperty("tableSchemaJson") Optional tableSchemaJson, + @JsonProperty("partitionFieldIds") Optional> partitionFieldIds, + @JsonProperty("equalityFieldIds") Optional> equalityFieldIds) { super(schemaName, icebergTableName.getTableName()); @@ -45,6 +50,8 @@ public IcebergTableHandle( this.snapshotSpecified = snapshotSpecified; this.predicate = requireNonNull(predicate, "predicate is null"); this.tableSchemaJson = requireNonNull(tableSchemaJson, "tableSchemaJson is null"); + this.partitionFieldIds = requireNonNull(partitionFieldIds, "partitionFieldIds is null"); + this.equalityFieldIds = requireNonNull(equalityFieldIds, "equalityFieldIds is null"); } @JsonProperty @@ -71,6 +78,18 @@ public Optional getTableSchemaJson() return tableSchemaJson; } + @JsonProperty + public Optional> getPartitionSpecId() + { + return partitionFieldIds; + } + + @JsonProperty + public Optional> getEqualityFieldIds() + { + return equalityFieldIds; + } + @Override public boolean equals(Object o) { @@ -86,13 +105,14 @@ public boolean equals(Object o) Objects.equals(icebergTableName, that.icebergTableName) && snapshotSpecified == that.snapshotSpecified && Objects.equals(predicate, that.predicate) && - Objects.equals(tableSchemaJson, that.tableSchemaJson); + Objects.equals(tableSchemaJson, that.tableSchemaJson) && + Objects.equals(equalityFieldIds, that.equalityFieldIds); } @Override public int hashCode() { - return Objects.hash(getSchemaName(), icebergTableName, predicate, snapshotSpecified, tableSchemaJson); + return Objects.hash(getSchemaName(), icebergTableName, predicate, snapshotSpecified, tableSchemaJson, equalityFieldIds); } @Override diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java index cbaabe3011694..87ec01e3c037d 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java @@ -127,6 +127,10 @@ public static IcebergTableName from(String name) } } + if (!type.isPublic()) { + throw new PrestoException(NOT_SUPPORTED, format("Internal Iceberg table name (type '%s'): %s", typeString, name)); + } + Optional version = Optional.empty(); Optional changelogEndVersion = Optional.empty(); if (type == DATA || type == PARTITIONS || type == MANIFESTS || type == FILES) { diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java index 853c38fa48a49..cd46f344f359a 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableType.java @@ -15,12 +15,26 @@ public enum IcebergTableType { - DATA, - HISTORY, - SNAPSHOTS, - MANIFESTS, - PARTITIONS, - FILES, - PROPERTIES, - CHANGELOG + DATA(true), + HISTORY(true), + SNAPSHOTS(true), + MANIFESTS(true), + PARTITIONS(true), + FILES(true), + PROPERTIES(true), + CHANGELOG(true), + EQUALITY_DELETES(true), + DATA_WITHOUT_EQUALITY_DELETES(false); + + private final boolean isPublic; + + IcebergTableType(boolean isPublic) + { + this.isPublic = isPublic; + } + + public boolean isPublic() + { + return isPublic; + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index 989baec687622..adb8db329c851 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -43,9 +43,13 @@ import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.HistoryEntry; @@ -62,6 +66,7 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; @@ -78,6 +83,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -86,6 +92,7 @@ import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.facebook.presto.common.type.BigintType.BIGINT; @@ -147,6 +154,7 @@ import static java.lang.Long.parseLong; import static java.lang.Math.toIntExact; import static java.lang.String.format; +import static java.util.Collections.emptyIterator; import static java.util.Comparator.comparing; import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE; import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP; @@ -710,18 +718,24 @@ public static Map getPartitionKeys(ContentScanTask fieldToIndex = getIdentityPartitions(spec); + return getPartitionKeys(spec, partition); + } + + public static Map getPartitionKeys(PartitionSpec spec, StructLike partition) + { Map partitionKeys = new HashMap<>(); - fieldToIndex.forEach((field, index) -> { - int id = field.sourceId(); + int index = 0; + for (PartitionField field : spec.fields()) { + int sourceId = field.sourceId(); String colName = field.name(); - org.apache.iceberg.types.Type type = spec.schema().findType(id); + org.apache.iceberg.types.Type sourceType = spec.schema().findType(sourceId); + org.apache.iceberg.types.Type type = field.transform().getResultType(sourceType); Class javaClass = type.typeId().javaClass(); Object value = partition.get(index, javaClass); if (value == null) { - partitionKeys.put(id, new HivePartitionKey(colName, Optional.empty())); + partitionKeys.put(field.fieldId(), new HivePartitionKey(colName, Optional.empty())); } else { HivePartitionKey partitionValue; @@ -732,9 +746,13 @@ public static Map getPartitionKeys(ContentScanTask properties, Iceberg properties.put(IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, String.valueOf(icebergConfig.getManifestCacheMaxContentLength())); properties.put(IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, String.valueOf(icebergConfig.getManifestCacheExpireDuration())); } + + public static long getDataSequenceNumber(ContentFile file) + { + if (file.dataSequenceNumber() != null) { + return file.dataSequenceNumber(); + } + return file.fileSequenceNumber(); + } + + /** + * Provides the delete files that need to be applied to the given table snapshot. + * + * @param table The table to provide deletes for + * @param snapshot The snapshot id to use + * @param filter Filters to apply during planning + * @param requestedPartitionSpec If provided, only delete files for this partition spec will be provided + * @param requestedSchema If provided, only delete files with this schema will be provided + */ + public static CloseableIterable getDeleteFiles(Table table, + long snapshot, + TupleDomain filter, + Optional> requestedPartitionSpec, + Optional> requestedSchema) + { + Expression filterExpression = toIcebergExpression(filter); + CloseableIterable fileTasks = table.newScan().useSnapshot(snapshot).filter(filterExpression).planFiles(); + + return new CloseableIterable() + { + @Override + public void close() + throws IOException + { + fileTasks.close(); + } + + @Override + public CloseableIterator iterator() + { + return new DeleteFilesIterator(table.specs(), fileTasks.iterator(), requestedPartitionSpec, requestedSchema); + } + }; + } + + private static class DeleteFilesIterator + implements CloseableIterator + { + private final Set seenFiles = new HashSet<>(); + private final Map partitionSpecsById; + private CloseableIterator fileTasks; + private final Optional> requestedPartitionSpec; + private final Optional> requestedSchema; + private Iterator currentDeletes = emptyIterator(); + private DeleteFile currentFile; + + private DeleteFilesIterator(Map partitionSpecsById, + CloseableIterator fileTasks, + Optional> requestedPartitionSpec, + Optional> requestedSchema) + { + this.partitionSpecsById = partitionSpecsById; + this.fileTasks = fileTasks; + this.requestedPartitionSpec = requestedPartitionSpec; + this.requestedSchema = requestedSchema; + } + + @Override + public boolean hasNext() + { + return currentFile != null || advance(); + } + + private boolean advance() + { + currentFile = null; + while (currentFile == null && (currentDeletes.hasNext() || fileTasks.hasNext())) { + if (!currentDeletes.hasNext()) { + currentDeletes = fileTasks.next().deletes().iterator(); + } + while (currentDeletes.hasNext()) { + DeleteFile deleteFile = currentDeletes.next(); + if (shouldIncludeFile(deleteFile)) { + // If there is a requested schema only include files that match it + if (seenFiles.add(deleteFile.path().toString())) { + currentFile = deleteFile; + return true; + } + } + } + } + return false; + } + + @Override + public DeleteFile next() + { + DeleteFile result = currentFile; + advance(); + return result; + } + + private boolean shouldIncludeFile(DeleteFile file) + { + boolean matchesPartition = !requestedPartitionSpec.isPresent() || + requestedPartitionSpec.get().equals(partitionSpecsById.get(file.specId()).fields().stream().map(PartitionField::fieldId).collect(Collectors.toSet())); + return matchesPartition && (file.content() == FileContent.POSITION_DELETES || + !requestedSchema.isPresent() || requestedSchema.get().equals(ImmutableSet.copyOf(file.equalityFieldIds()))); + } + + @Override + public void close() + throws IOException + { + fileTasks.close(); + // TODO: remove this after org.apache.iceberg.io.CloseableIterator'withClose + // correct release resources holds by iterator. + // (and make it final) + fileTasks = CloseableIterator.empty(); + } + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java index 5afacd9527a98..f068d7ecd0659 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java @@ -29,8 +29,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.theta.CompactSketch; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.ContentScanTask; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.GenericBlobMetadata; import org.apache.iceberg.GenericStatisticsFile; import org.apache.iceberg.HasTableOperations; @@ -138,42 +139,12 @@ private TableStatistics makeTableStatistics(IcebergTableHandle tableHandle, Cons .filter(column -> !identityPartitionIds.contains(column.fieldId()) && column.type().isPrimitiveType()) .collect(toImmutableList()); - TableScan tableScan = icebergTable.newScan() - .filter(toIcebergExpression(intersection)) - .select(selectedColumns.stream().map(IcebergColumnHandle::getName).collect(Collectors.toList())) - .useSnapshot(tableHandle.getIcebergTableName().getSnapshotId().get()) - .includeColumnStats(); - - Partition summary = null; - try (CloseableIterable fileScanTasks = tableScan.planFiles()) { - for (FileScanTask fileScanTask : fileScanTasks) { - DataFile dataFile = fileScanTask.file(); - - if (summary == null) { - summary = new Partition( - idToTypeMapping, - nonPartitionPrimitiveColumns, - dataFile.partition(), - dataFile.recordCount(), - dataFile.fileSizeInBytes(), - toMap(idToTypeMapping, dataFile.lowerBounds()), - toMap(idToTypeMapping, dataFile.upperBounds()), - dataFile.nullValueCounts(), - dataFile.columnSizes()); - } - else { - summary.incrementFileCount(); - summary.incrementRecordCount(dataFile.recordCount()); - summary.incrementSize(dataFile.fileSizeInBytes()); - updateSummaryMin(summary, partitionFields, toMap(idToTypeMapping, dataFile.lowerBounds()), dataFile.nullValueCounts(), dataFile.recordCount()); - updateSummaryMax(summary, partitionFields, toMap(idToTypeMapping, dataFile.upperBounds()), dataFile.nullValueCounts(), dataFile.recordCount()); - summary.updateNullCount(dataFile.nullValueCounts()); - updateColumnSizes(summary, dataFile.columnSizes()); - } - } + Partition summary; + if (tableHandle.getIcebergTableName().getTableType() == IcebergTableType.EQUALITY_DELETES) { + summary = getEqualityDeleteTableSummary(tableHandle, intersection, idToTypeMapping, nonPartitionPrimitiveColumns, partitionFields); } - catch (IOException e) { - throw new UncheckedIOException(e); + else { + summary = getDataTableSummary(tableHandle, selectedColumns, intersection, idToTypeMapping, nonPartitionPrimitiveColumns, partitionFields); } if (summary == null) { @@ -211,6 +182,75 @@ private TableStatistics makeTableStatistics(IcebergTableHandle tableHandle, Cons return result.build(); } + private Partition getDataTableSummary(IcebergTableHandle tableHandle, + List selectedColumns, + TupleDomain intersection, + Map idToTypeMapping, + List nonPartitionPrimitiveColumns, + List partitionFields) + { + TableScan tableScan = icebergTable.newScan() + .filter(toIcebergExpression(intersection)) + .select(selectedColumns.stream().map(IcebergColumnHandle::getName).collect(Collectors.toList())) + .useSnapshot(tableHandle.getIcebergTableName().getSnapshotId().get()) + .includeColumnStats(); + + CloseableIterable> files = CloseableIterable.transform(tableScan.planFiles(), ContentScanTask::file); + return getSummaryFromFiles(files, idToTypeMapping, nonPartitionPrimitiveColumns, partitionFields); + } + + private Partition getEqualityDeleteTableSummary(IcebergTableHandle tableHandle, + TupleDomain intersection, + Map idToTypeMapping, + List nonPartitionPrimitiveColumns, + List partitionFields) + { + CloseableIterable deleteFiles = IcebergUtil.getDeleteFiles(icebergTable, + tableHandle.getIcebergTableName().getSnapshotId().get(), + intersection, + tableHandle.getPartitionSpecId(), + tableHandle.getEqualityFieldIds()); + CloseableIterable> files = CloseableIterable.transform(deleteFiles, deleteFile -> deleteFile); + return getSummaryFromFiles(files, idToTypeMapping, nonPartitionPrimitiveColumns, partitionFields); + } + + private Partition getSummaryFromFiles(CloseableIterable> files, + Map idToTypeMapping, + List nonPartitionPrimitiveColumns, + List partitionFields) + { + Partition summary = null; + try (CloseableIterable> filesHolder = files) { + for (ContentFile contentFile : filesHolder) { + if (summary == null) { + summary = new Partition( + idToTypeMapping, + nonPartitionPrimitiveColumns, + contentFile.partition(), + contentFile.recordCount(), + contentFile.fileSizeInBytes(), + toMap(idToTypeMapping, contentFile.lowerBounds()), + toMap(idToTypeMapping, contentFile.upperBounds()), + contentFile.nullValueCounts(), + contentFile.columnSizes()); + } + else { + summary.incrementFileCount(); + summary.incrementRecordCount(contentFile.recordCount()); + summary.incrementSize(contentFile.fileSizeInBytes()); + updateSummaryMin(summary, partitionFields, toMap(idToTypeMapping, contentFile.lowerBounds()), contentFile.nullValueCounts(), contentFile.recordCount()); + updateSummaryMax(summary, partitionFields, toMap(idToTypeMapping, contentFile.upperBounds()), contentFile.nullValueCounts(), contentFile.recordCount()); + summary.updateNullCount(contentFile.nullValueCounts()); + updateColumnSizes(summary, contentFile.columnSizes()); + } + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + return summary; + } + public static void writeTableStatistics(NodeVersion nodeVersion, IcebergTableHandle tableHandle, Table icebergTable, ConnectorSession session, Collection computedStatistics) { new TableStatisticsMaker(icebergTable, session).writeTableStatistics(nodeVersion, tableHandle, computedStatistics); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitSource.java index 5e0052ec5e811..cc301573cbeef 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitSource.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitSource.java @@ -43,6 +43,7 @@ import static com.facebook.presto.hive.HiveCommonSessionProperties.getNodeSelectionStrategy; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_CANNOT_OPEN_SPLIT; import static com.facebook.presto.iceberg.IcebergUtil.getColumns; +import static com.facebook.presto.iceberg.IcebergUtil.getDataSequenceNumber; import static com.facebook.presto.iceberg.IcebergUtil.getPartitionKeys; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.google.common.collect.Iterators.limit; @@ -128,6 +129,7 @@ private IcebergSplit splitFromContentScanTask(ContentScanTask task, Ch Optional.of(new ChangelogSplitInfo(changeTask.operation(), changeTask.changeOrdinal(), changeTask.commitSnapshotId(), - columnHandles))); + columnHandles)), + getDataSequenceNumber(task.file())); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/equalitydeletes/EqualityDeletesSplitSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/equalitydeletes/EqualityDeletesSplitSource.java new file mode 100644 index 0000000000000..948779f33b2a0 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/equalitydeletes/EqualityDeletesSplitSource.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.equalitydeletes; + +import com.facebook.presto.iceberg.IcebergSplit; +import com.facebook.presto.iceberg.IcebergUtil; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SplitWeight; +import com.facebook.presto.spi.connector.ConnectorPartitionHandle; +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.CloseableIterator; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import static com.facebook.presto.hive.HiveCommonSessionProperties.getNodeSelectionStrategy; +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; +import static com.facebook.presto.iceberg.IcebergUtil.getPartitionKeys; +import static com.google.common.collect.Iterators.limit; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.CompletableFuture.completedFuture; + +public class EqualityDeletesSplitSource + implements ConnectorSplitSource +{ + private final ConnectorSession session; + private final Map specById; + private CloseableIterator deleteFiles; + + public EqualityDeletesSplitSource( + ConnectorSession session, + Table table, + CloseableIterable deleteFiles) + { + this.session = requireNonNull(session, "session is null"); + requireNonNull(table, "table is null"); + requireNonNull(deleteFiles, "deleteFiles is null"); + this.specById = table.specs(); + this.deleteFiles = CloseableIterable.filter(deleteFiles, deleteFile -> deleteFile.content() == FileContent.EQUALITY_DELETES).iterator(); + } + + @Override + public boolean isFinished() + { + return !deleteFiles.hasNext(); + } + + @Override + public CompletableFuture getNextBatch(ConnectorPartitionHandle partitionHandle, int maxSize) + { + ImmutableList.Builder splits = new ImmutableList.Builder<>(); + Iterator iterator = limit(deleteFiles, maxSize); + iterator.forEachRemaining(manifestReader -> { + splits.add(toIcebergSplit(manifestReader)); + }); + return completedFuture(new ConnectorSplitBatch(splits.build(), isFinished())); + } + + @Override + public void close() + { + try { + deleteFiles.close(); + // TODO: remove this after org.apache.iceberg.io.CloseableIterator'withClose + // correct release resources holds by iterator. + // (and make it final) + deleteFiles = CloseableIterator.empty(); + } + catch (IOException e) { + throw new PrestoException(ICEBERG_FILESYSTEM_ERROR, e); + } + } + + private ConnectorSplit toIcebergSplit(DeleteFile manifesReader) + { + return splitFromDeleteFile(manifesReader); + } + + private IcebergSplit splitFromDeleteFile(DeleteFile deleteFile) + { + return new IcebergSplit( + deleteFile.path().toString(), + 0, + deleteFile.fileSizeInBytes(), + deleteFile.format(), + ImmutableList.of(), + getPartitionKeys(specById.get(deleteFile.specId()), deleteFile.partition()), + getNodeSelectionStrategy(session), + SplitWeight.standard(), + ImmutableList.of(), + Optional.empty(), + IcebergUtil.getDataSequenceNumber(deleteFile)); + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java new file mode 100644 index 0000000000000..a411566ff8d9c --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java @@ -0,0 +1,465 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.optimizer; + +import com.facebook.presto.common.function.OperatorType; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.common.type.BooleanType; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.iceberg.ColumnIdentity; +import com.facebook.presto.iceberg.IcebergAbstractMetadata; +import com.facebook.presto.iceberg.IcebergColumnHandle; +import com.facebook.presto.iceberg.IcebergMetadataColumn; +import com.facebook.presto.iceberg.IcebergTableHandle; +import com.facebook.presto.iceberg.IcebergTableName; +import com.facebook.presto.iceberg.IcebergTableType; +import com.facebook.presto.iceberg.IcebergTransactionManager; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.ConnectorPlanRewriter; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.ConnectorJoinNode; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.PARTITION_KEY; +import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.REGULAR; +import static com.facebook.presto.iceberg.IcebergColumnHandle.DATA_SEQUENCE_NUMBER_COLUMN_HANDLE; +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; +import static com.facebook.presto.iceberg.IcebergMetadataColumn.DATA_SEQUENCE_NUMBER; +import static com.facebook.presto.iceberg.IcebergSessionProperties.isDeleteToJoinPushdownEnabled; +import static com.facebook.presto.iceberg.IcebergUtil.getDeleteFiles; +import static com.facebook.presto.iceberg.IcebergUtil.getIcebergTable; +import static com.facebook.presto.iceberg.TypeConverter.toPrestoType; +import static com.facebook.presto.spi.ConnectorPlanRewriter.rewriteWith; +import static java.util.Objects.requireNonNull; + +/** + *

This optimizer implements equality deletes as a join, rather than having each split read the delete files and apply them. + * This approach significantly enhances performance for equality deletes, as most delete files will apply to most splits, + * and opening the delete file in each split incurs considerable overhead. Usually, the delete files are relatively small + * and can be broadcast easily. Each delete file may have a different schema, though typically there will be only a few delete + * schemas, often just one (the primary key).

+ * + *

For example, consider the following query: + * SELECT * FROM table; + * With 2 delete schemas: (pk), (orderid), the query will be transformed into: + * 

+ * SELECT "$data_sequence_number", * FROM table
+ * LEFT JOIN "table$equality_deletes1" d1 ON left.pk = d1.pk AND left."$data_sequence_number" < d1."$data_sequence_number" -- Find deletes by schema 1
+ * LEFT JOIN "table$equality_deletes2" d2 ON left.orderid = d1.orderid AND left."$data_sequence_number" < d2."$data_sequence_number" -- Find deletes by schema 2
+ * WHERE COALESCE(d1."$data_sequence_number", d2."data_sequence_number") IS NULL -- None of the delete files had a delete for this row
+ *
+ * Note that table$equality_deletes1 and table$equality_deletes2 are different tables, each containing only the delete files with the schema for this join.

+ */ + +public class IcebergEqualityDeleteAsJoin + implements ConnectorPlanOptimizer +{ + private final StandardFunctionResolution functionResolution; + private final IcebergTransactionManager transactionManager; + private final TypeManager typeManager; + + IcebergEqualityDeleteAsJoin(StandardFunctionResolution functionResolution, + IcebergTransactionManager transactionManager, + TypeManager typeManager) + { + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + public PlanNode optimize(PlanNode maxSubplan, ConnectorSession session, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator) + { + if (!isDeleteToJoinPushdownEnabled(session)) { + return maxSubplan; + } + return rewriteWith(new DeleteAsJoinRewriter(functionResolution, + transactionManager, idAllocator, session, typeManager, variableAllocator), maxSubplan); + } + + private static class DeleteAsJoinRewriter + extends ConnectorPlanRewriter + { + private final ConnectorSession session; + private final StandardFunctionResolution functionResolution; + private final PlanNodeIdAllocator idAllocator; + private final IcebergTransactionManager transactionManager; + private final TypeManager typeManager; + private final VariableAllocator variableAllocator; + + public DeleteAsJoinRewriter( + StandardFunctionResolution functionResolution, + IcebergTransactionManager transactionManager, + PlanNodeIdAllocator idAllocator, + ConnectorSession session, + TypeManager typeManager, + VariableAllocator variableAllocator) + { + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); + this.session = requireNonNull(session, "session is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.variableAllocator = requireNonNull(variableAllocator, "variableAllocator is null"); + } + + @Override + public PlanNode visitTableScan(TableScanNode node, RewriteContext context) + { + TableHandle table = node.getTable(); + IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table.getConnectorHandle(); + IcebergTableName tableName = icebergTableHandle.getIcebergTableName(); + if (!tableName.getSnapshotId().isPresent() || tableName.getTableType() != IcebergTableType.DATA) { + // Node is already optimized or not ready for planning + return node; + } + + IcebergAbstractMetadata metadata = (IcebergAbstractMetadata) transactionManager.get(table.getTransaction()); + Table icebergTable = getIcebergTable(metadata, session, icebergTableHandle.getSchemaTableName()); + + // Collect info about each unique delete schema to join by + ImmutableMap, DeleteSetInfo> deleteSchemas = collectDeleteInformation(icebergTable, icebergTableHandle, tableName.getSnapshotId().get()); + + if (deleteSchemas.isEmpty()) { + // no equality deletes + return node; + } + + // Add all the fields required by the join that were not added by the user's query + ImmutableMap unselectedAssignments = createAssignmentsForUnselectedFields(node, deleteSchemas, icebergTable); + TableScanNode updatedTableScan = createNewRoot(node, icebergTableHandle, tableName, unselectedAssignments, table); + + Map reverseAssignmentsMap = updatedTableScan + .getAssignments() + .entrySet() + .stream() + .collect(Collectors.toMap(assignment -> ((IcebergColumnHandle) (assignment.getValue())).getId(), Map.Entry::getKey)); + + List deleteVersionColumns = new ArrayList<>(); + PlanNode parentNode = updatedTableScan; + // For each unique delete schema add a join that applies those equality deletes + for (Map.Entry, DeleteSetInfo> entry : deleteSchemas.entrySet()) { + DeleteSetInfo deleteGroupInfo = entry.getValue(); + + List deleteFields = deleteGroupInfo + .equalityFieldIds + .stream() + .map(fieldId -> icebergTable.schema().findField(fieldId)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + VariableReferenceExpression joinSequenceNumber = toVariableReference(DATA_SEQUENCE_NUMBER_COLUMN_HANDLE); + deleteVersionColumns.add(joinSequenceNumber); + ImmutableMap deleteColumnAssignments = ImmutableMap.builder() + .putAll(deleteGroupInfo.allFields(icebergTable.schema()).stream().collect(Collectors.toMap(this::toVariableReference, this::toIcebergColumnHandle))) + .put(joinSequenceNumber, DATA_SEQUENCE_NUMBER_COLUMN_HANDLE) + .build(); + + // ON source.delete_column = deletes.delete_column, ... + Set clauses = deleteColumnAssignments + .entrySet() + .stream() + .filter(assignment -> !IcebergMetadataColumn.isMetadataColumnId(((IcebergColumnHandle) (assignment.getValue())).getId())) + .map(assignment -> { + VariableReferenceExpression left = reverseAssignmentsMap.get(((IcebergColumnHandle) (assignment.getValue())).getId()); + VariableReferenceExpression right = assignment.getKey(); + return new EquiJoinClause(left, right); + }).collect(Collectors.toSet()); + + FunctionHandle lessThan = functionResolution.comparisonFunction(OperatorType.LESS_THAN, BigintType.BIGINT, BigintType.BIGINT); + + // AND source.$data_sequence_number < deletes.$data_sequence_number + RowExpression versionFilter = new CallExpression(lessThan.getName(), + lessThan, + BooleanType.BOOLEAN, + Collections.unmodifiableList(Arrays.asList(reverseAssignmentsMap.get(DATA_SEQUENCE_NUMBER.getId()), joinSequenceNumber))); + + TableScanNode deleteTableScan = createDeletesTableScan(deleteColumnAssignments, + icebergTableHandle, + tableName, + deleteFields, + table, + deleteGroupInfo); + + parentNode = new ConnectorJoinNode(idAllocator.getNextId(), + Arrays.asList(parentNode, deleteTableScan), + Optional.empty(), + JoinType.LEFT, + clauses, + Sets.newHashSet(versionFilter), + Optional.empty(), // Allow stats to determine join distribution + Stream.concat(parentNode.getOutputVariables().stream(), deleteTableScan.getOutputVariables().stream()).collect(Collectors.toList())); + } + + FilterNode filter = new FilterNode(Optional.empty(), idAllocator.getNextId(), Optional.empty(), parentNode, + new SpecialFormExpression(SpecialFormExpression.Form.IS_NULL, BooleanType.BOOLEAN, + new SpecialFormExpression(SpecialFormExpression.Form.COALESCE, BigintType.BIGINT, deleteVersionColumns))); + + return filter; + } + + private static ImmutableMap, DeleteSetInfo> collectDeleteInformation(Table icebergTable, + IcebergTableHandle icebergTableHandle, + long snapshotId) + { + // Delete schemas can repeat, so using a normal hashmap to dedup, will be converted to immutable at the end of the function. + HashMap, DeleteSetInfo> deleteInformations = new HashMap<>(); + try (CloseableIterator files = + getDeleteFiles(icebergTable, snapshotId, icebergTableHandle.getPredicate(), Optional.empty(), Optional.empty()).iterator()) { + files.forEachRemaining(delete -> { + if (delete.content() == FileContent.EQUALITY_DELETES) { + ImmutableMap.Builder partitionFieldsBuilder = new ImmutableMap.Builder<>(); + PartitionSpec partitionSpec = icebergTable.specs().get(delete.specId()); + Types.StructType partitionType = partitionSpec.partitionType(); + // PartitionField ids are unique across the entire table in v2. We can assume we are in v2 since v1 doesn't have equality deletes + partitionSpec.fields().forEach(f -> partitionFieldsBuilder.put(f.fieldId(), new PartitionFieldInfo(partitionType.field(f.fieldId()), f))); + ImmutableMap partitionFields = partitionFieldsBuilder.build(); + HashSet result = new HashSet<>(); + result.addAll(delete.equalityFieldIds()); + result.addAll(partitionFields.keySet()); + deleteInformations.put(ImmutableSet.copyOf(result), new DeleteSetInfo(partitionFields, delete.equalityFieldIds())); + } + }); + } + catch (IOException e) { + throw new PrestoException(ICEBERG_FILESYSTEM_ERROR, "Failed to read equality delete information", e); + } + return ImmutableMap.copyOf(deleteInformations); + } + + private TableScanNode createDeletesTableScan(ImmutableMap deleteColumnAssignments, + IcebergTableHandle icebergTableHandle, + IcebergTableName tableName, + List deleteFields, + TableHandle table, + DeleteSetInfo deleteInfo) + { + List outputs = deleteColumnAssignments.keySet().asList(); + IcebergTableHandle deletesTableHandle = new IcebergTableHandle(icebergTableHandle.getSchemaName(), + new IcebergTableName(tableName.getTableName(), + IcebergTableType.EQUALITY_DELETES, // Read equality deletes instead of data + tableName.getSnapshotId(), + Optional.empty()), + icebergTableHandle.isSnapshotSpecified(), + icebergTableHandle.getPredicate(), + Optional.of(SchemaParser.toJson(new Schema(deleteFields))), + Optional.of(deleteInfo.partitionFields.keySet()), // Enforce reading only delete files that match this schema + Optional.of(deleteInfo.equalityFieldIds)); + + return new TableScanNode(Optional.empty(), + idAllocator.getNextId(), + new TableHandle(table.getConnectorId(), deletesTableHandle, table.getTransaction(), table.getLayout(), table.getDynamicFilter()), + outputs, + deleteColumnAssignments, + TupleDomain.all(), + TupleDomain.all()); + } + + /** + * - Updates table handle to DATA_WITHOUT_EQUALITY_DELETES since the page source for this node should now not apply equality deletes. + * - Adds extra assignments and outputs that are needed by the join + */ + private TableScanNode createNewRoot(TableScanNode node, IcebergTableHandle icebergTableHandle, IcebergTableName tableName, ImmutableMap unselectedAssignments, TableHandle table) + { + IcebergTableHandle updatedHandle = new IcebergTableHandle(icebergTableHandle.getSchemaName(), + new IcebergTableName(tableName.getTableName(), + IcebergTableType.DATA_WITHOUT_EQUALITY_DELETES, // Don't apply equality deletes in the split + tableName.getSnapshotId(), + tableName.getChangelogEndSnapshot()), + icebergTableHandle.isSnapshotSpecified(), + icebergTableHandle.getPredicate(), + icebergTableHandle.getTableSchemaJson(), + icebergTableHandle.getPartitionSpecId(), + icebergTableHandle.getEqualityFieldIds()); + + VariableReferenceExpression dataSequenceNumberVariableReference = toVariableReference(DATA_SEQUENCE_NUMBER_COLUMN_HANDLE); + ImmutableMap.Builder assignmentsBuilder = ImmutableMap.builder() + .put(dataSequenceNumberVariableReference, DATA_SEQUENCE_NUMBER_COLUMN_HANDLE) + .putAll(unselectedAssignments) + .putAll(node.getAssignments()); + ImmutableList.Builder outputsBuilder = ImmutableList.builder(); + outputsBuilder.addAll(node.getOutputVariables()); + if (!node.getAssignments().containsKey(dataSequenceNumberVariableReference)) { + outputsBuilder.add(dataSequenceNumberVariableReference); + } + outputsBuilder.addAll(unselectedAssignments.keySet()); + + return new TableScanNode(node.getSourceLocation(), + node.getId(), + Optional.of(node), + new TableHandle(table.getConnectorId(), updatedHandle, table.getTransaction(), table.getLayout(), table.getDynamicFilter()), + outputsBuilder.build(), + assignmentsBuilder.build(), + node.getTableConstraints(), + node.getCurrentConstraint(), + node.getEnforcedConstraint()); + } + + /** + * Calculate required fields that the user didn't include in his select statement and add assignments for them to add to the table scan + */ + private ImmutableMap createAssignmentsForUnselectedFields(TableScanNode node, + ImmutableMap, DeleteSetInfo> deleteInfos, + Table icebergTable) + { + Set selectedFields = node.getAssignments().values().stream().map(f -> ((IcebergColumnHandle) f).getId()).collect(Collectors.toSet()); + Set unselectedFields = Sets.difference(deleteInfos.keySet().stream().reduce(Sets::union).orElseGet(Collections::emptySet), selectedFields); + ImmutableMap.Builder unselectedAssignmentsBuilder = ImmutableMap.builder(); + Map partitionFields = deleteInfos.values().stream() + .flatMap(info -> info.getPartitionFields().entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (existing, replacement) -> existing)); + unselectedFields + .forEach(fieldId -> { + if (partitionFields.containsKey(fieldId)) { + PartitionFieldInfo partitionFieldInfo = partitionFields.get(fieldId); + PartitionField partitionField = partitionFieldInfo.getPartitionField(); + Types.NestedField sourceField = icebergTable.schema().findField(partitionField.sourceId()); + if (!partitionField.transform().isIdentity()) { + Type partitionFieldType = partitionField.transform().getResultType(sourceField.type()); + VariableReferenceExpression variableReference = variableAllocator.newVariable(partitionField.name(), toPrestoType(partitionFieldType, typeManager)); + IcebergColumnHandle columnHandle = new IcebergColumnHandle( + ColumnIdentity.createColumnIdentity(partitionField.name(), partitionField.fieldId(), partitionFieldType), + toPrestoType(partitionFieldType, typeManager), + Optional.empty(), + PARTITION_KEY); + unselectedAssignmentsBuilder.put(variableReference, columnHandle); + } + else if (!selectedFields.contains(sourceField.fieldId())) { + unselectedAssignmentsBuilder.put( + variableAllocator.newVariable(sourceField.name(), toPrestoType(sourceField.type(), typeManager)), + IcebergColumnHandle.create(sourceField, typeManager, REGULAR)); + } + } + else { + Types.NestedField schemaField = icebergTable.schema().findField(fieldId); + unselectedAssignmentsBuilder.put( + variableAllocator.newVariable(schemaField.name(), toPrestoType(schemaField.type(), typeManager)), + IcebergColumnHandle.create(schemaField, typeManager, REGULAR)); + } + }); + return unselectedAssignmentsBuilder.build(); + } + + private VariableReferenceExpression toVariableReference(IcebergColumnHandle columnHandle) + { + return variableAllocator.newVariable(columnHandle.getName(), columnHandle.getType()); + } + + private IcebergColumnHandle toIcebergColumnHandle(Types.NestedField field) + { + ColumnIdentity columnIdentity = new ColumnIdentity(field.fieldId(), field.name(), ColumnIdentity.TypeCategory.PRIMITIVE, Collections.emptyList()); + return new IcebergColumnHandle(columnIdentity, toPrestoType(field.type(), typeManager), Optional.empty(), REGULAR); + } + + private VariableReferenceExpression toVariableReference(Types.NestedField field) + { + return variableAllocator.newVariable(field.name(), toPrestoType(field.type(), typeManager)); + } + + private static class PartitionFieldInfo + { + private final Types.NestedField nestedField; + private final PartitionField partitionField; + + private PartitionFieldInfo(Types.NestedField nestedField, PartitionField partitionField) + { + this.nestedField = nestedField; + this.partitionField = partitionField; + } + + public PartitionField getPartitionField() + { + return partitionField; + } + } + + private static class DeleteSetInfo + { + private final ImmutableMap partitionFields; + private final Set equalityFieldIds; + + private DeleteSetInfo(ImmutableMap partitionFields, + List equalityFieldIds) + { + this.partitionFields = requireNonNull(partitionFields, "partitionFields is null"); + this.equalityFieldIds = ImmutableSet.copyOf(requireNonNull(equalityFieldIds, "equalityFieldIds is null")); + } + + public ImmutableMap getPartitionFields() + { + return partitionFields; + } + + public List allFields(Schema schema) + { + return Stream.concat(equalityFieldIds + .stream() + .map(schema::findField), + partitionFields + .values() + .stream() + .map(partitionFieldInfo -> { + if (partitionFieldInfo.partitionField.transform().isIdentity()) { + return schema.findField(partitionFieldInfo.partitionField.sourceId()); + } + return partitionFieldInfo.nestedField; + })).collect(Collectors.toList()); + } + } + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java index 79832fd866336..2382d9337de67 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java @@ -134,7 +134,9 @@ public PlanNode visitFilter(FilterNode filter, RewriteContext context) oldTableHandle.getIcebergTableName(), oldTableHandle.isSnapshotSpecified(), simplifiedColumnDomain, - oldTableHandle.getTableSchemaJson()); + oldTableHandle.getTableSchemaJson(), + oldTableHandle.getPartitionSpecId(), + oldTableHandle.getEqualityFieldIds()); TableScanNode newTableScan = new TableScanNode( tableScan.getSourceLocation(), tableScan.getId(), diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java index 5485bc9c7d76c..c2efe7ad00c0b 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java @@ -31,6 +31,7 @@ public class IcebergPlanOptimizerProvider implements ConnectorPlanOptimizerProvider { private final Set planOptimizers; + private final Set logicalPlanOptimizers; @Inject public IcebergPlanOptimizerProvider( @@ -49,12 +50,16 @@ public IcebergPlanOptimizerProvider( new IcebergPlanOptimizer(functionResolution, rowExpressionService, transactionManager), new IcebergFilterPushdown(rowExpressionService, functionResolution, functionMetadataManager, transactionManager, typeManager), new IcebergParquetDereferencePushDown(transactionManager, rowExpressionService, typeManager)); + this.logicalPlanOptimizers = ImmutableSet.builder() + .addAll(this.planOptimizers) + .add(new IcebergEqualityDeleteAsJoin(functionResolution, transactionManager, typeManager)) + .build(); } @Override public Set getLogicalPlanOptimizers() { - return planOptimizers; + return logicalPlanOptimizers; } @Override diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java index 7b9f8870c5510..09f1a26845321 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java @@ -15,6 +15,8 @@ import com.facebook.presto.Session; import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.block.Block; +import com.facebook.presto.common.block.BlockBuilder; import com.facebook.presto.common.transaction.TransactionId; import com.facebook.presto.hive.HdfsConfiguration; import com.facebook.presto.hive.HdfsConfigurationInitializer; @@ -62,6 +64,7 @@ import org.apache.iceberg.deletes.PositionDeleteWriter; import org.apache.iceberg.hadoop.HadoopOutputFile; import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.types.Types; import org.apache.iceberg.util.TableScanUtil; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -74,6 +77,8 @@ import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -81,11 +86,16 @@ import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.TimeZoneKey.UTC_KEY; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; import static com.facebook.presto.iceberg.IcebergQueryRunner.TEST_CATALOG_DIRECTORY; import static com.facebook.presto.iceberg.IcebergQueryRunner.TEST_DATA_DIRECTORY; +import static com.facebook.presto.iceberg.IcebergSessionProperties.DELETE_AS_JOIN_REWRITE_ENABLED; import static com.facebook.presto.iceberg.IcebergSessionProperties.STATISTIC_SNAPSHOT_RECORD_DIFFERENCE_WEIGHT; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; @@ -859,24 +869,43 @@ public void testTableWithPositionDelete(String fileFormat) assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE nationkey not in (0 ,8)"); } - @Test(dataProvider = "fileFormat") - public void testTableWithEqualityDelete(String fileFormat) + @DataProvider(name = "equalityDeleteOptions") + public Object[][] equalityDeleteDataProvider() + { + return new Object[][] { + {"PARQUET", false}, + {"PARQUET", true}, + {"ORC", false}, + {"ORC", true}}; + } + + private Session deleteAsJoinEnabled(boolean joinRewriteEnabled) + { + return Session.builder(getQueryRunner().getDefaultSession()) + .setCatalogSessionProperty(ICEBERG_CATALOG, DELETE_AS_JOIN_REWRITE_ENABLED, String.valueOf(joinRewriteEnabled)) + .build(); + } + + @Test(dataProvider = "equalityDeleteOptions") + public void testTableWithEqualityDelete(String fileFormat, boolean joinRewriteEnabled) throws Exception { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); String tableName = "test_v2_equality_delete" + randomTableSuffix(); - assertUpdate("CREATE TABLE " + tableName + " with (format = '" + fileFormat + "') AS SELECT * FROM tpch.tiny.nation", 25); + assertUpdate(session, "CREATE TABLE " + tableName + " with (format = '" + fileFormat + "') AS SELECT * FROM tpch.tiny.nation", 25); Table icebergTable = updateTable(tableName); writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 1L)); testCheckDeleteFiles(icebergTable, 1, ImmutableList.of(EQUALITY_DELETES)); - assertQuery("SELECT * FROM " + tableName, "SELECT * FROM nation WHERE regionkey != 1"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1"); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE regionkey != 1"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1"); } - @Test(dataProvider = "fileFormat") - public void testTableWithPositionDeleteAndEqualityDelete(String fileFormat) + @Test(dataProvider = "equalityDeleteOptions") + public void testTableWithPositionDeleteAndEqualityDelete(String fileFormat, boolean joinRewriteEnabled) throws Exception { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); String tableName = "test_v2_row_delete_" + randomTableSuffix(); assertUpdate("CREATE TABLE " + tableName + " with (format = '" + fileFormat + "') AS SELECT * FROM tpch.tiny.nation order by nationkey", 25); Table icebergTable = updateTable(tableName); @@ -884,43 +913,157 @@ public void testTableWithPositionDeleteAndEqualityDelete(String fileFormat) writePositionDeleteToNationTable(icebergTable, dataFilePath, 0); testCheckDeleteFiles(icebergTable, 1, ImmutableList.of(POSITION_DELETES)); - assertQuery("SELECT count(*) FROM " + tableName, "VALUES 24"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE nationkey != 0"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 24"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE nationkey != 0"); writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 1L)); testCheckDeleteFiles(icebergTable, 2, ImmutableList.of(POSITION_DELETES, EQUALITY_DELETES)); - assertQuery("SELECT * FROM " + tableName, "SELECT * FROM nation WHERE regionkey != 1 AND nationkey != 0"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1 AND nationkey != 0"); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE regionkey != 1 AND nationkey != 0"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1 AND nationkey != 0"); } - @Test(dataProvider = "fileFormat") - public void testTableWithPositionDeletesAndEqualityDeletes(String fileFormat) + @Test(dataProvider = "equalityDeleteOptions") + public void testEqualityDeletesWithPartitions(String fileFormat, boolean joinRewriteEnabled) throws Exception { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); + String tableName = "test_v2_row_delete_" + randomTableSuffix(); + assertUpdate("CREATE TABLE " + tableName + " with (format = '" + fileFormat + "', partitioning = ARRAY['nationkey']) AS SELECT * FROM tpch.tiny.nation order by nationkey", 25); + Table icebergTable = updateTable(tableName); + + List partitions = Arrays.asList(1L, 2L, 3L, 17L, 24L); + for (long nationKey : partitions) { + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 1L), ImmutableMap.of("nationkey", nationKey)); + } + testCheckDeleteFiles(icebergTable, partitions.size(), partitions.stream().map(i -> EQUALITY_DELETES).collect(Collectors.toList())); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE regionkey != 1"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1"); + assertQuery(session, "SELECT name FROM " + tableName, "SELECT name FROM nation WHERE regionkey != 1"); + } + + @Test(dataProvider = "equalityDeleteOptions") + public void testEqualityDeletesWithHiddenPartitions(String fileFormat, boolean joinRewriteEnabled) + throws Exception + { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); + String tableName = "test_v2_row_delete_" + randomTableSuffix(); + assertUpdate("CREATE TABLE " + tableName + " with (format = '" + fileFormat + "', partitioning = ARRAY['bucket(nationkey,100)']) AS SELECT * FROM tpch.tiny.nation order by nationkey", 25); + Table icebergTable = updateTable(tableName); + + PartitionTransforms.ColumnTransform columnTransform = PartitionTransforms.getColumnTransform(icebergTable.spec().fields().get(0), BIGINT); + BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(5); + List partitions = Arrays.asList(1L, 2L, 3L, 17L, 24L); + partitions.forEach(p -> BIGINT.writeLong(builder, p)); + Block partitionsBlock = columnTransform.getTransform().apply(builder.build()); + for (int i = 0; i < partitionsBlock.getPositionCount(); i++) { + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 1L), ImmutableMap.of("nationkey_bucket", partitionsBlock.getInt(i))); + String updatedPartitions = partitions.stream().limit(i + 1).map(Object::toString).collect(Collectors.joining(",", "(", ")")); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE NOT(regionkey = 1 AND nationkey IN " + updatedPartitions + ")"); + } + testCheckDeleteFiles(icebergTable, partitions.size(), partitions.stream().map(i -> EQUALITY_DELETES).collect(Collectors.toList())); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE regionkey != 1"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1"); + assertQuery(session, "SELECT name FROM " + tableName, "SELECT name FROM nation WHERE regionkey != 1"); + } + + @Test(dataProvider = "equalityDeleteOptions") + public void testEqualityDeletesWithCompositeKey(String fileFormat, boolean joinRewriteEnabled) + throws Exception + { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); String tableName = "test_v2_row_delete_" + randomTableSuffix(); assertUpdate("CREATE TABLE " + tableName + " with (format = '" + fileFormat + "') AS SELECT * FROM tpch.tiny.nation order by nationkey", 25); Table icebergTable = updateTable(tableName); + + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 0L, "name", "ALGERIA")); + testCheckDeleteFiles(icebergTable, 1, Stream.generate(() -> EQUALITY_DELETES).limit(1).collect(Collectors.toList())); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE NOT(regionkey = 0 AND name = 'ALGERIA')"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE NOT(regionkey = 0 AND name = 'ALGERIA')"); + assertQuery(session, "SELECT name FROM " + tableName, "SELECT name FROM nation WHERE NOT(regionkey = 0 AND name = 'ALGERIA')"); + } + + @Test(dataProvider = "equalityDeleteOptions") + public void testEqualityDeletesWithMultipleDeleteSchemas(String fileFormat, boolean joinRewriteEnabled) + throws Exception + { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); + String tableName = "test_v2_row_delete_" + randomTableSuffix(); + assertUpdate("CREATE TABLE " + tableName + " with (format = '" + fileFormat + "') AS SELECT * FROM tpch.tiny.nation order by nationkey", 25); + Table icebergTable = updateTable(tableName); + + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 1L)); + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("nationkey", 10L)); + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 2L, "nationkey", 9L)); + testCheckDeleteFiles(icebergTable, 3, Stream.generate(() -> EQUALITY_DELETES).limit(3).collect(Collectors.toList())); + assertQuery(session, "SELECT \"$data_sequence_number\", regionkey, nationkey FROM \"" + tableName + "$equality_deletes\"", "VALUES (2, 1, null), (3, null, 10), (4, 2, 9)"); + assertQuery(session, "SELECT * FROM " + tableName, "SELECT * FROM nation WHERE NOT(regionkey = 2 AND nationkey = 9) AND nationkey <> 10 AND regionkey <> 1"); + assertQuery(session, "SELECT regionkey FROM " + tableName, "SELECT regionkey FROM nation WHERE NOT(regionkey = 2 AND nationkey = 9) AND nationkey <> 10 AND regionkey <> 1"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE NOT(regionkey = 2 AND nationkey = 9) AND nationkey <> 10 AND regionkey <> 1"); + } + + @Test(dataProvider = "equalityDeleteOptions") + public void testTableWithPositionDeletesAndEqualityDeletes(String fileFormat, boolean joinRewriteEnabled) + throws Exception + { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); + String tableName = "test_v2_row_delete_" + randomTableSuffix(); + assertUpdate(session, "CREATE TABLE " + tableName + " with (format = '" + fileFormat + "') AS SELECT * FROM tpch.tiny.nation order by nationkey", 25); + Table icebergTable = updateTable(tableName); String dataFilePath = (String) computeActual("SELECT file_path FROM \"" + tableName + "$files\" LIMIT 1").getOnlyValue(); writePositionDeleteToNationTable(icebergTable, dataFilePath, 0); testCheckDeleteFiles(icebergTable, 1, ImmutableList.of(POSITION_DELETES)); - assertQuery("SELECT count(*) FROM " + tableName, "VALUES 24"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE nationkey != 0"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 24"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE nationkey != 0"); writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 1L)); testCheckDeleteFiles(icebergTable, 2, ImmutableList.of(POSITION_DELETES, EQUALITY_DELETES)); - assertQuery("SELECT count(*) FROM " + tableName, "VALUES 19"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1 AND nationkey != 0"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 19"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1 AND nationkey != 0"); writePositionDeleteToNationTable(icebergTable, dataFilePath, 7); testCheckDeleteFiles(icebergTable, 3, ImmutableList.of(POSITION_DELETES, POSITION_DELETES, EQUALITY_DELETES)); - assertQuery("SELECT count(*) FROM " + tableName, "VALUES 18"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1 AND nationkey NOT IN (0, 7)"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 18"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey != 1 AND nationkey NOT IN (0, 7)"); writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("regionkey", 2L)); testCheckDeleteFiles(icebergTable, 4, ImmutableList.of(POSITION_DELETES, POSITION_DELETES, EQUALITY_DELETES, EQUALITY_DELETES)); - assertQuery("SELECT count(*) FROM " + tableName, "VALUES 13"); - assertQuery("SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey NOT IN (1, 2) AND nationkey NOT IN (0, 7)"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 13"); + assertQuery(session, "SELECT nationkey FROM " + tableName, "SELECT nationkey FROM nation WHERE regionkey NOT IN (1, 2) AND nationkey NOT IN (0, 7)"); + } + + @Test(dataProvider = "equalityDeleteOptions") + public void testEqualityDeletesWithHiddenPartitionsEvolution(String fileFormat, boolean joinRewriteEnabled) + throws Exception + { + Session session = deleteAsJoinEnabled(joinRewriteEnabled); + String tableName = "test_v2_row_delete_" + randomTableSuffix(); + assertUpdate("CREATE TABLE " + tableName + "(a int, b varchar) WITH (format = '" + fileFormat + "')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, '1001'), (2, '1002'), (3, '1003')", 3); + + Table icebergTable = updateTable(tableName); + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("a", 2)); + assertQuery(session, "SELECT * FROM " + tableName, "VALUES (1, '1001'), (3, '1003')"); + + assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN c int WITH (partitioning = 'bucket(2)')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (6, '1004', 1), (6, '1006', 2)", 2); + icebergTable = updateTable(tableName); + PartitionTransforms.ColumnTransform columnTransform = PartitionTransforms.getColumnTransform(icebergTable.spec().fields().get(0), INTEGER); + BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(1); + List partitions = Arrays.asList(1, 2); + partitions.forEach(p -> BIGINT.writeLong(builder, p)); + Block partitionsBlock = columnTransform.getTransform().apply(builder.build()); + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("a", 6, "c", 2), + ImmutableMap.of("c_bucket", partitionsBlock.getInt(0))); + assertQuery(session, "SELECT * FROM " + tableName, "VALUES (1, '1001', NULL), (3, '1003', NULL), (6, '1004', 1)"); + + assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN d varchar WITH (partitioning = 'truncate(2)')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (6, '1004', 1, 'th001'), (6, '1006', 2, 'th002')", 2); + icebergTable = updateTable(tableName); + writeEqualityDeleteToNationTable(icebergTable, ImmutableMap.of("a", 6, "c", 1), + ImmutableMap.of("c_bucket", partitionsBlock.getInt(0), "d_trunc", "th")); + testCheckDeleteFiles(icebergTable, 3, ImmutableList.of(EQUALITY_DELETES, EQUALITY_DELETES, EQUALITY_DELETES)); + assertQuery(session, "SELECT * FROM " + tableName, "VALUES (1, '1001', NULL, NULL), (3, '1003', NULL, NULL), (6, '1004', 1, NULL), (6, '1006', 2, 'th002')"); } private void testCheckDeleteFiles(Table icebergTable, int expectedSize, List expectedFileContent) @@ -967,19 +1110,31 @@ private void writePositionDeleteToNationTable(Table icebergTable, String dataFil private void writeEqualityDeleteToNationTable(Table icebergTable, Map overwriteValues) throws Exception + { + writeEqualityDeleteToNationTable(icebergTable, overwriteValues, Collections.emptyMap()); + } + + private void writeEqualityDeleteToNationTable(Table icebergTable, Map overwriteValues, Map partitionValues) + throws Exception { File metastoreDir = getDistributedQueryRunner().getCoordinator().getDataDirectory().toFile(); org.apache.hadoop.fs.Path metadataDir = new org.apache.hadoop.fs.Path(metastoreDir.toURI()); String deleteFileName = "delete_file_" + UUID.randomUUID(); FileSystem fs = getHdfsEnvironment().getFileSystem(new HdfsContext(SESSION), metadataDir); - Schema deleteRowSchema = icebergTable.schema().select("regionkey"); - EqualityDeleteWriter writer = Parquet.writeDeletes(HadoopOutputFile.fromPath(new org.apache.hadoop.fs.Path(metadataDir, deleteFileName), fs)) + Schema deleteRowSchema = icebergTable.schema().select(overwriteValues.keySet()); + Parquet.DeleteWriteBuilder writerBuilder = Parquet.writeDeletes(HadoopOutputFile.fromPath(new org.apache.hadoop.fs.Path(metadataDir, deleteFileName), fs)) .forTable(icebergTable) .rowSchema(deleteRowSchema) .createWriterFunc(GenericParquetWriter::buildWriter) - .equalityFieldIds(deleteRowSchema.findField("regionkey").fieldId()) - .overwrite() - .buildEqualityWriter(); + .equalityFieldIds(deleteRowSchema.columns().stream().map(Types.NestedField::fieldId).collect(Collectors.toList())) + .overwrite(); + + if (!partitionValues.isEmpty()) { + GenericRecord partitionData = GenericRecord.create(icebergTable.spec().partitionType()); + writerBuilder.withPartition(partitionData.copy(partitionValues)); + } + + EqualityDeleteWriter writer = writerBuilder.buildEqualityWriter(); Record dataDelete = GenericRecord.create(deleteRowSchema); try (Closeable ignored = writer) { diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java index 8a3c0d9a26da9..7d84025f72ba5 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java @@ -53,6 +53,7 @@ public void testDefaults() .setParquetDereferencePushdownEnabled(true) .setMergeOnReadModeEnabled(true) .setPushdownFilterEnabled(false) + .setDeleteAsJoinRewriteEnabled(true) .setManifestCachingEnabled(false) .setFileIOImpl(HadoopFileIO.class.getName()) .setMaxManifestCacheSize(IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT) @@ -77,6 +78,7 @@ public void testExplicitPropertyMappings() .put("iceberg.statistic-snapshot-record-difference-weight", "1.0") .put("iceberg.hive-statistics-merge-strategy", "USE_NDV") .put("iceberg.pushdown-filter-enabled", "true") + .put("iceberg.delete-as-join-rewrite-enabled", "false") .put("iceberg.io.manifest.cache-enabled", "true") .put("iceberg.io-impl", "com.facebook.presto.iceberg.HdfsFileIO") .put("iceberg.io.manifest.cache.max-total-bytes", "1048576000") @@ -98,6 +100,7 @@ public void testExplicitPropertyMappings() .setMergeOnReadModeEnabled(false) .setHiveStatisticsMergeStrategy(USE_NDV) .setPushdownFilterEnabled(true) + .setDeleteAsJoinRewriteEnabled(false) .setManifestCachingEnabled(true) .setFileIOImpl("com.facebook.presto.iceberg.HdfsFileIO") .setMaxManifestCacheSize(1048576000) diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java index 09bca1ef506ad..c6c4d80896abc 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java @@ -21,6 +21,7 @@ import com.facebook.presto.common.predicate.ValueSet; import com.facebook.presto.common.transaction.TransactionId; import com.facebook.presto.iceberg.IcebergColumnHandle; +import com.facebook.presto.iceberg.IcebergMetadataColumn; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.Constraint; @@ -319,7 +320,9 @@ private TableHandle getTableHandle(String tableName, Session session) private Map getColumnHandles(String tableName, Session session) { - return getQueryRunner().getMetadata().getColumnHandles(session, getTableHandle(tableName, session)); + return getQueryRunner().getMetadata().getColumnHandles(session, getTableHandle(tableName, session)).entrySet().stream() + .filter(entry -> !IcebergMetadataColumn.isMetadataColumnId(((IcebergColumnHandle) (entry.getValue())).getId())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } static void assertStatValuePresent(StatsSchema column, MaterializedResult result, Set columnNames) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java index 1c858bdee1ac5..58d294fa44aec 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java @@ -17,6 +17,7 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.OutputNode; import com.facebook.presto.spi.plan.PlanNode; @@ -183,7 +184,7 @@ public PlanCostEstimate visitJoin(JoinNode node, Void context) node, node.getLeft(), node.getRight(), - Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED))); + Objects.equals(node.getDistributionType(), Optional.of(JoinDistributionType.REPLICATED))); return costForLookupJoin(node, localCost); } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java index f84ccfc6a7b65..314117b1dab12 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java @@ -17,6 +17,7 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.sql.planner.iterative.GroupReference; @@ -128,7 +129,7 @@ public LocalCostEstimate visitJoin(JoinNode node, Void context) node.getLeft(), node.getRight(), stats, - Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED)), + Objects.equals(node.getDistributionType(), Optional.of(JoinDistributionType.REPLICATED)), taskCountEstimator.estimateSourceDistributedTaskCount()); } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java index 2c3f694d5d9e9..bf040ea81fa74 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java @@ -15,12 +15,12 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.SymbolReference; import com.facebook.presto.util.MoreMath; @@ -414,7 +414,7 @@ private PlanNodeStatsEstimate crossJoinStats(JoinNode node, PlanNodeStatsEstimat return normalizer.normalize(builder.build()); } - private List flippedCriteria(JoinNode node) + private List flippedCriteria(JoinNode node) { return node.getCriteria().stream() .map(EquiJoinClause::flip) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalJoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalJoinNode.java index af73714f21ca2..7856af02d09a2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalJoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalJoinNode.java @@ -14,11 +14,12 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,8 +36,8 @@ public class CanonicalJoinNode extends PlanNode { private final List sources; - private final JoinNode.Type type; - private final Set criteria; + private final JoinType type; + private final Set criteria; private final Set filters; private final List outputVariables; @@ -44,8 +45,8 @@ public class CanonicalJoinNode public CanonicalJoinNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("sources") List sources, - @JsonProperty("type") JoinNode.Type type, - @JsonProperty("criteria") Set criteria, + @JsonProperty("type") JoinType type, + @JsonProperty("criteria") Set criteria, @JsonProperty("filter") Set filters, @JsonProperty("outputVariables") List outputVariables) { @@ -65,13 +66,13 @@ public List getSources() } @JsonProperty - public JoinNode.Type getType() + public JoinType getType() { return type; } @JsonProperty - public Set getCriteria() + public Set getCriteria() { return criteria; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java index 84578460246b4..c970afe0ab006 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java @@ -25,7 +25,9 @@ import com.facebook.presto.spi.plan.AggregationNode.GroupingSetDescriptor; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.Ordering; @@ -271,12 +273,12 @@ public Optional visitJoin(JoinNode node, Context context) if (strategy == DEFAULT) { return Optional.empty(); } - if (node.getType().equals(JoinNode.Type.RIGHT)) { + if (node.getType().equals(JoinType.RIGHT)) { return visitJoin(node.flipChildren(), context); } List sources = new ArrayList<>(); ImmutableList.Builder allFilters = ImmutableList.builder(); - ImmutableList.Builder criterias = ImmutableList.builder(); + ImmutableList.Builder criterias = ImmutableList.builder(); Stack stack = new Stack<>(); stack.push(node); @@ -288,7 +290,7 @@ public Optional visitJoin(JoinNode node, Context context) if (top.getFilter().isPresent()) { List filters = extractConjuncts(top.getFilter().get()); filters.forEach(filter -> { - Optional criteria = toEquiJoinClause(filter); + Optional criteria = toEquiJoinClause(filter); criteria.ifPresent(criterias::add); allFilters.add(filter); }); @@ -306,7 +308,7 @@ && shouldMergeJoinNodes(node.getType())) { } // Sort sources if all are INNER, or full outer join of 2 nodes - if (shouldMergeJoinNodes(node.getType()) || (node.getType().equals(JoinNode.Type.FULL) && sources.size() == 2)) { + if (shouldMergeJoinNodes(node.getType()) || (node.getType().equals(JoinType.FULL) && sources.size() == 2)) { Optional> sourceIndexes = orderSources(sources); if (!sourceIndexes.isPresent()) { return Optional.empty(); @@ -322,9 +324,9 @@ && shouldMergeJoinNodes(node.getType())) { } newSources.add(newSource.get()); } - Set newCriterias = criterias.build().stream() + Set newCriterias = criterias.build().stream() .map(criteria -> canonicalize(criteria, context)) - .sorted(comparing(JoinNode.EquiJoinClause::toString)) + .sorted(comparing(EquiJoinClause::toString)) .collect(toCollection(LinkedHashSet::new)); Set newFilters = allFilters.build().stream() .map(filter -> inlineAndCanonicalize(context.getExpressions(), filter)) @@ -1134,9 +1136,9 @@ public Optional visitTableScan(TableScanNode node, Context context) return Optional.of(canonicalPlan); } - private boolean shouldMergeJoinNodes(JoinNode.Type type) + private boolean shouldMergeJoinNodes(JoinType type) { - return type.equals(JoinNode.Type.INNER); + return type.equals(JoinType.INNER); } private VariableReferenceExpression rename(VariableReferenceExpression variable, String nameHint, Context context) @@ -1156,14 +1158,14 @@ private String writeValueAsString(Object object) } } - private static JoinNode.EquiJoinClause canonicalize(JoinNode.EquiJoinClause criteria, Context context) + private static EquiJoinClause canonicalize(EquiJoinClause criteria, Context context) { VariableReferenceExpression left = inlineAndCanonicalize(context.getExpressions(), criteria.getLeft()); VariableReferenceExpression right = inlineAndCanonicalize(context.getExpressions(), criteria.getRight()); - return left.compareTo(right) > 0 ? new JoinNode.EquiJoinClause(left, right) : new JoinNode.EquiJoinClause(right, left); + return left.compareTo(right) > 0 ? new EquiJoinClause(left, right) : new EquiJoinClause(right, left); } - private static Optional toEquiJoinClause(RowExpression expression) + private static Optional toEquiJoinClause(RowExpression expression) { if (!(expression instanceof CallExpression)) { return Optional.empty(); @@ -1178,7 +1180,7 @@ private static Optional toEquiJoinClause(RowExpression return Optional.empty(); } - return Optional.of(new JoinNode.EquiJoinClause( + return Optional.of(new EquiJoinClause( (VariableReferenceExpression) callExpression.getArguments().get(0), (VariableReferenceExpression) callExpression.getArguments().get(1))); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java index 0a2868d82068f..56e389a9d60ac 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; @@ -338,7 +339,7 @@ public RowExpression visitSpatialJoin(SpatialJoinNode node, Void context) } } - private RowExpression toRowExpression(JoinNode.EquiJoinClause equiJoinClause) + private RowExpression toRowExpression(EquiJoinClause equiJoinClause) { return buildEqualsExpression(functionManger, equiJoinClause.getLeft(), equiJoinClause.getRight()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java index 23629746c5554..c13c5b8b6e81c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; @@ -85,7 +86,7 @@ public GroupedExecutionTagger.GroupedExecutionProperties visitJoin(JoinNode node return GroupedExecutionTagger.GroupedExecutionProperties.notCapable(); } - if ((node.getType() == JoinNode.Type.RIGHT || node.getType() == JoinNode.Type.FULL) && !right.currentNodeCapable) { + if ((node.getType() == JoinType.RIGHT || node.getType() == JoinType.FULL) && !right.currentNodeCapable) { // For a plan like this, if the fragment participates in grouped execution, // the LookupOuterOperator corresponding to the RJoin will not work execute properly. // diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index e727f9141de29..231de8c651773 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -146,7 +146,9 @@ import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.OrderingScheme; @@ -309,6 +311,10 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.INTERMEDIATE; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.spi.plan.ProjectNode.Locality.REMOTE; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; @@ -321,10 +327,6 @@ import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SCALED_WRITER_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; import static com.facebook.presto.sql.tree.SortItem.Ordering.DESCENDING; @@ -1940,9 +1942,9 @@ public PhysicalOperation visitJoin(JoinNode node, LocalExecutionPlanContext cont return createNestedLoopJoin(node, context); } - List clauses = node.getCriteria(); - List leftVariables = Lists.transform(clauses, JoinNode.EquiJoinClause::getLeft); - List rightVariables = Lists.transform(clauses, JoinNode.EquiJoinClause::getRight); + List clauses = node.getCriteria(); + List leftVariables = Lists.transform(clauses, EquiJoinClause::getLeft); + List rightVariables = Lists.transform(clauses, EquiJoinClause::getRight); switch (node.getType()) { case INNER: @@ -2428,7 +2430,7 @@ private JoinBridgeManager createLookupSourceFact filter -> factoriesBuilder.add(createDynamicFilterSourceOperatorFactory(filter, node.getId(), buildSource, buildContext))); // Determine if planning broadcast join - Optional distributionType = node.getDistributionType(); + Optional distributionType = node.getDistributionType(); boolean isBroadcastJoin = distributionType.isPresent() && distributionType.get() == REPLICATED; HashBuilderOperatorFactory hashBuilderOperatorFactory = new HashBuilderOperatorFactory( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java index f3db7bde150a4..fbb8aabe0c2d0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java @@ -71,12 +71,12 @@ import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.ConnectorId.isInternalSystemConnector; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.getSourceLocation; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.variable; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java index 2d2f39ce6cf1d..82242e8e648f6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.CteReferenceNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; @@ -301,7 +302,7 @@ protected RelationPlan visitJoin(Join node, SqlPlannerContext context) .addAll(rightPlan.getFieldMappings()) .build(); - ImmutableList.Builder equiClauses = ImmutableList.builder(); + ImmutableList.Builder equiClauses = ImmutableList.builder(); List complexJoinExpressions = new ArrayList<>(); List postInnerJoinConditions = new ArrayList<>(); @@ -369,7 +370,7 @@ else if (firstDependencies.stream().allMatch(right::canResolve) && secondDepende VariableReferenceExpression leftVariable = leftPlanBuilder.translateToVariable(leftComparisonExpressions.get(i)); VariableReferenceExpression rightVariable = rightPlanBuilder.translateToVariable(rightComparisonExpressions.get(i)); - equiClauses.add(new JoinNode.EquiJoinClause(leftVariable, rightVariable)); + equiClauses.add(new EquiJoinClause(leftVariable, rightVariable)); } else { Expression leftExpression = leftPlanBuilder.rewrite(leftComparisonExpressions.get(i)); @@ -517,7 +518,7 @@ If casts are redundant (due to column type and common type being equal), Analysis.JoinUsingAnalysis joinAnalysis = analysis.getJoinUsing(node); - ImmutableList.Builder clauses = ImmutableList.builder(); + ImmutableList.Builder clauses = ImmutableList.builder(); Map leftJoinColumns = new HashMap<>(); Map rightJoinColumns = new HashMap<>(); @@ -557,7 +558,7 @@ If casts are redundant (due to column type and common type being equal), context)); rightJoinColumns.put(identifier, rightOutput); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput)); + clauses.add(new EquiJoinClause(leftOutput, rightOutput)); } ProjectNode leftCoercion = new ProjectNode(idAllocator.getNextId(), left.getRoot(), leftCoercions.build()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java index e9e2a12dc4336..a699a35133fec 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java @@ -14,10 +14,11 @@ package com.facebook.presto.sql.planner.iterative.properties; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LogicalProperties; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.relational.FunctionResolution; import java.util.ArrayList; @@ -28,16 +29,16 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.iterative.properties.Key.getNormalizedKey; import static com.facebook.presto.sql.planner.iterative.properties.KeyProperty.combineKey; import static com.facebook.presto.sql.planner.iterative.properties.KeyProperty.combineKeys; import static com.facebook.presto.sql.planner.iterative.properties.KeyProperty.concatKeyProperty; import static com.facebook.presto.sql.planner.iterative.properties.KeyProperty.getNormalizedKeyProperty; import static com.facebook.presto.sql.planner.iterative.properties.MaxCardProperty.multiplyMaxCard; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -369,8 +370,8 @@ public static LogicalPropertiesImpl distinctLimitProperties(LogicalPropertiesImp */ public static LogicalPropertiesImpl joinProperties(LogicalPropertiesImpl leftProperties, LogicalPropertiesImpl rightProperties, - List equijoinPredicates, - JoinNode.Type joinType, + List equijoinPredicates, + JoinType joinType, Optional filterPredicate, FunctionResolution functionResolution) { @@ -381,8 +382,8 @@ public static LogicalPropertiesImpl joinProperties(LogicalPropertiesImpl leftPro // first determine if the join is n to 1 and/or 1 to n boolean nToOne = false; boolean oneToN = false; - Set rightJoinVariables = equijoinPredicates.stream().map(JoinNode.EquiJoinClause::getRight).collect(Collectors.toSet()); - Set leftJoinVariables = equijoinPredicates.stream().map(JoinNode.EquiJoinClause::getLeft).collect(Collectors.toSet()); + Set rightJoinVariables = equijoinPredicates.stream().map(EquiJoinClause::getRight).collect(Collectors.toSet()); + Set leftJoinVariables = equijoinPredicates.stream().map(EquiJoinClause::getLeft).collect(Collectors.toSet()); //if n-to-1 inner or left join then propagate left source keys and maxcard if ((rightProperties.maxCardProperty.isAtMostOne() || (!rightJoinVariables.isEmpty() && rightProperties.isDistinct(rightJoinVariables))) && @@ -422,7 +423,7 @@ public static LogicalPropertiesImpl joinProperties(LogicalPropertiesImpl leftPro //update equivalence classes with equijoin predicates, note that if nulls are injected, equivalence does not hold propagate if (joinType == INNER) { - for (JoinNode.EquiJoinClause equiJoinClause : equijoinPredicates) { + for (EquiJoinClause equiJoinClause : equijoinPredicates) { equivalenceClassProperty = equivalenceClassProperty.combineWith(equiJoinClause.getLeft(), equiJoinClause.getRight()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddNotNullFiltersToJoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddNotNullFiltersToJoinNode.java index a37e0bf527ee9..71d1f25852663 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddNotNullFiltersToJoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddNotNullFiltersToJoinNode.java @@ -21,6 +21,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.IntermediateFormExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -132,12 +133,12 @@ public Result apply(JoinNode joinNode, Captures captures, Context context) joinNode.getDynamicFilters())); } - private Collection extractNotNullVariables(List joinCriteria, Optional joinFilter, + private Collection extractNotNullVariables(List joinCriteria, Optional joinFilter, List candidates, JoinNotNullInferenceStrategy notNullInferenceStrategy) { RowExpression combinedFilter = TRUE_CONSTANT; - for (JoinNode.EquiJoinClause criteria : joinCriteria) { + for (EquiJoinClause criteria : joinCriteria) { combinedFilter = and(combinedFilter, criteria.getLeft()); combinedFilter = and(combinedFilter, criteria.getRight()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayContainsToInnerJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayContainsToInnerJoin.java index c66c69484b85c..4ee94f2f6fb35 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayContainsToInnerJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayContainsToInnerJoin.java @@ -19,7 +19,9 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -76,7 +78,7 @@ public class CrossJoinWithArrayContainsToInnerJoin { private static final Capture CHILD = newCapture(); private static final Pattern PATTERN = filter() - .with(source().matching(join().matching(x -> x.getType().equals(JoinNode.Type.INNER) && x.getCriteria().isEmpty()).capturedAs(CHILD))); + .with(source().matching(join().matching(x -> x.getType().equals(JoinType.INNER) && x.getCriteria().isEmpty()).capturedAs(CHILD))); private final FunctionAndTypeManager functionAndTypeManager; @@ -112,7 +114,7 @@ public boolean isEnabled(Session session) public Result apply(FilterNode node, Captures captures, Context context) { JoinNode joinNode = captures.get(CHILD); - if (!(joinNode.getType().equals(JoinNode.Type.INNER) && joinNode.getCriteria().isEmpty())) { + if (!(joinNode.getType().equals(JoinType.INNER) && joinNode.getCriteria().isEmpty())) { return Result.empty(); } List leftInput = joinNode.getLeft().getOutputVariables(); @@ -148,7 +150,7 @@ public Result apply(FilterNode node, Captures captures, Context context) ImmutableMap.of(arrayDistinctVariable, ImmutableList.of(unnestVariable)), Optional.empty()); - JoinNode.EquiJoinClause equiJoinClause = arrayAtLeftInput ? new JoinNode.EquiJoinClause(unnestVariable, elementVar) : new JoinNode.EquiJoinClause(elementVar, unnestVariable); + EquiJoinClause equiJoinClause = arrayAtLeftInput ? new EquiJoinClause(unnestVariable, elementVar) : new EquiJoinClause(elementVar, unnestVariable); JoinNode newJoinNode = new JoinNode(joinNode.getSourceLocation(), context.getIdAllocator().getNextId(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayNotContainsToAntiJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayNotContainsToAntiJoin.java index 22046db2f6923..01e5d51f49cc4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayNotContainsToAntiJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithArrayNotContainsToAntiJoin.java @@ -21,7 +21,9 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; @@ -128,7 +130,7 @@ public Result apply(FilterNode node, Captures captures, Context context) { JoinNode joinNode = captures.get(JOIN); - if (!(joinNode.getType().equals(JoinNode.Type.INNER) && joinNode.getCriteria().isEmpty())) { + if (!(joinNode.getType().equals(JoinType.INNER) && joinNode.getCriteria().isEmpty())) { return Result.empty(); } List leftColumns = joinNode.getLeft().getOutputVariables(); @@ -182,13 +184,13 @@ public Result apply(FilterNode node, Captures captures, Context context) // if element is not a VariableReferenceExpression, push the expression into a Project node so the variable can be used in equijoins checkState(element instanceof VariableReferenceExpression, "Argument to CONTAINS is not a column"); - JoinNode.EquiJoinClause equiJoinClause = new JoinNode.EquiJoinClause((VariableReferenceExpression) element, unnestVariable); + EquiJoinClause equiJoinClause = new EquiJoinClause((VariableReferenceExpression) element, unnestVariable); List newOutputColumns = Stream.concat(newLeftNode.getOutputVariables().stream(), unnest.getOutputVariables().stream()).collect(toImmutableList()); JoinNode newJoinNode = new JoinNode(joinNode.getSourceLocation(), context.getIdAllocator().getNextId(), - JoinNode.Type.LEFT, + JoinType.LEFT, newLeftNode, unnest, ImmutableList.of(equiJoinClause), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithOrFilterToInnerJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithOrFilterToInnerJoin.java index aa093850ccc8e..a219a986507de 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithOrFilterToInnerJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CrossJoinWithOrFilterToInnerJoin.java @@ -23,7 +23,9 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; @@ -120,7 +122,7 @@ public class CrossJoinWithOrFilterToInnerJoin private static final Capture CHILD = newCapture(); private static final Pattern PATTERN = filter() - .with(source().matching(join().matching(x -> x.getType().equals(JoinNode.Type.INNER) && x.getCriteria().isEmpty()).capturedAs(CHILD))); + .with(source().matching(join().matching(x -> x.getType().equals(JoinType.INNER) && x.getCriteria().isEmpty()).capturedAs(CHILD))); private final FunctionAndTypeManager functionAndTypeManager; @@ -230,7 +232,7 @@ else if (candidate.contains(argument1)) { public Result apply(FilterNode filterNode, Captures captures, Context context) { JoinNode joinNode = captures.get(CHILD); - if (!(joinNode.getType().equals(JoinNode.Type.INNER) && joinNode.getCriteria().isEmpty())) { + if (!(joinNode.getType().equals(JoinType.INNER) && joinNode.getCriteria().isEmpty())) { return Result.empty(); } RowExpression candidateOrExpressions = getCandidateOrExpression(filterNode.getPredicate(), joinNode.getLeft().getOutputVariables(), joinNode.getRight().getOutputVariables()); @@ -270,8 +272,8 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) joinNode.getType(), leftJoinInput.getNode(), rightJoinInput.getNode(), - ImmutableList.of(new JoinNode.EquiJoinClause(leftJoinInput.getJoinKey(), rightJoinInput.getJoinKey()), - new JoinNode.EquiJoinClause(leftJoinInput.getUnnestIndex(), rightJoinInput.getUnnestIndex())), + ImmutableList.of(new EquiJoinClause(leftJoinInput.getJoinKey(), rightJoinInput.getJoinKey()), + new EquiJoinClause(leftJoinInput.getUnnestIndex(), rightJoinInput.getUnnestIndex())), joinOutput.build(), joinNode.getFilter(), Optional.empty(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java index e57025d7259a7..8b255d2781316 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java @@ -45,12 +45,12 @@ import static com.facebook.presto.SystemSessionProperties.isSizeBasedJoinDistributionTypeEnabled; import static com.facebook.presto.SystemSessionProperties.isUseBroadcastJoinWhenBuildSizeSmallProbeSizeUnknownEnabled; import static com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges.calculateJoinCostWithoutOutput; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.AUTOMATIC; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.isBelowBroadcastLimit; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.isSmallerThanThreshold; import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java index 163807543c4cc..f94c42c2d5665 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java @@ -17,7 +17,9 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -163,12 +165,12 @@ public static PlanNode buildJoinTree(List expectedO PlanNode rightNode = graph.getNode(joinOrder.get(i)); alreadyJoinedNodes.add(rightNode.getId()); - ImmutableList.Builder criteria = ImmutableList.builder(); + ImmutableList.Builder criteria = ImmutableList.builder(); for (JoinGraph.Edge edge : graph.getEdges(rightNode)) { PlanNode targetNode = edge.getTargetNode(); if (alreadyJoinedNodes.contains(targetNode.getId())) { - criteria.add(new JoinNode.EquiJoinClause( + criteria.add(new EquiJoinClause( edge.getTargetVariable(), edge.getSourceVariable())); } @@ -177,7 +179,7 @@ public static PlanNode buildJoinTree(List expectedO result = new JoinNode( result.getSourceLocation(), idAllocator.getNextId(), - JoinNode.Type.INNER, + JoinType.INNER, result, rightNode, criteria.build(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java index 8ba5e081ef7aa..fe812f9dae349 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java @@ -86,11 +86,11 @@ import static com.facebook.presto.spi.StandardErrorCode.INVALID_SPATIAL_PARTITIONING; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.VariablesExtractor.extractUnique; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; import static com.facebook.presto.sql.planner.plan.Patterns.filter; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.facebook.presto.sql.planner.plan.Patterns.source; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/JoinSwappingUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/JoinSwappingUtils.java index 058f185ac255b..803a08e2308a8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/JoinSwappingUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/JoinSwappingUtils.java @@ -18,6 +18,7 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -101,7 +102,7 @@ public static Optional createRuntimeSwappedJoinNode( // Add additional localExchange if the new build side does not satisfy the partitioning conditions. List buildJoinVariables = swapped.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .collect(toImmutableList()); PlanNode newRight = swapped.getRight(); if (!checkBuildSidePropertySatisfied(swapped.getRight(), buildJoinVariables, metadata, parser, lookup, session, variableAllocator)) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinNullFilterToSemiJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinNullFilterToSemiJoin.java index 206c1ba454c06..31aa5cbea0bff 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinNullFilterToSemiJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinNullFilterToSemiJoin.java @@ -20,6 +20,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; @@ -83,7 +84,7 @@ public class LeftJoinNullFilterToSemiJoin implements Rule { private static final Capture CHILD = newCapture(); - private static final Pattern PATTERN = filter().with(source().matching(join().matching(x -> x.getType().equals(JoinNode.Type.LEFT) + private static final Pattern PATTERN = filter().with(source().matching(join().matching(x -> x.getType().equals(JoinType.LEFT) && x.getCriteria().size() == 1 && !x.getFilter().isPresent() && x.getDynamicFilters().isEmpty()).capturedAs(CHILD))); private final FunctionAndTypeManager functionAndTypeManager; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinWithArrayContainsToEquiJoinCondition.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinWithArrayContainsToEquiJoinCondition.java index efccd0654c1ce..8b90cbfa0ec68 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinWithArrayContainsToEquiJoinCondition.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/LeftJoinWithArrayContainsToEquiJoinCondition.java @@ -18,6 +18,8 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -70,7 +72,7 @@ public class LeftJoinWithArrayContainsToEquiJoinCondition implements Rule { - private static final Pattern PATTERN = join().matching(x -> x.getType().equals(JoinNode.Type.LEFT) && x.getCriteria().isEmpty() && x.getFilter().isPresent()); + private static final Pattern PATTERN = join().matching(x -> x.getType().equals(JoinType.LEFT) && x.getCriteria().isEmpty() && x.getFilter().isPresent()); private final FunctionAndTypeManager functionAndTypeManager; private final RowExpressionDeterminismEvaluator determinismEvaluator; private final FunctionResolution functionResolution; @@ -136,7 +138,7 @@ public Result apply(JoinNode node, Captures captures, Context context) ImmutableMap.of(arrayFilterNullVariable, ImmutableList.of(unnestVariable)), Optional.empty()); - JoinNode.EquiJoinClause equiJoinClause = new JoinNode.EquiJoinClause(elementVariable, unnestVariable); + EquiJoinClause equiJoinClause = new EquiJoinClause(elementVariable, unnestVariable); return Result.ofPlanNode(new JoinNode(node.getSourceLocation(), context.getIdAllocator().getNextId(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneJoinChildrenColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneJoinChildrenColumns.java index c97348bba6840..ee4b5928ff9d5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneJoinChildrenColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneJoinChildrenColumns.java @@ -15,6 +15,7 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.JoinNode; @@ -61,7 +62,7 @@ public Result apply(JoinNode joinNode, Captures captures, Context context) .addAll(globallyUsableInputs) .addAll( joinNode.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getLeft) + .map(EquiJoinClause::getLeft) .iterator()) .addAll(joinNode.getLeftHashVariable().map(ImmutableSet::of).orElse(ImmutableSet.of())) .build(); @@ -70,7 +71,7 @@ public Result apply(JoinNode joinNode, Captures captures, Context context) .addAll(globallyUsableInputs) .addAll( joinNode.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .iterator()) .addAll(joinNode.getRightHashVariable().map(ImmutableSet::of).orElse(ImmutableSet.of())) .build(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java index e567f1900669d..b0d204395cae1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java @@ -24,6 +24,8 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.Ordering; import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; @@ -134,14 +136,14 @@ public Result apply(AggregationNode aggregation, Captures captures, Context cont JoinNode join = captures.get(JOIN); if (join.getFilter().isPresent() - || !(join.getType() == JoinNode.Type.LEFT || join.getType() == JoinNode.Type.RIGHT) + || !(join.getType() == JoinType.LEFT || join.getType() == JoinType.RIGHT) || !groupsOnAllColumns(aggregation, getOuterTable(join).getOutputVariables()) || !isDistinct(context.getLookup().resolve(getOuterTable(join)), context.getLookup()::resolve)) { return Result.empty(); } List groupingKeys = join.getCriteria().stream() - .map(join.getType() == JoinNode.Type.RIGHT ? JoinNode.EquiJoinClause::getLeft : JoinNode.EquiJoinClause::getRight) + .map(join.getType() == JoinType.RIGHT ? EquiJoinClause::getLeft : EquiJoinClause::getRight) .collect(toImmutableList()); AggregationNode rewrittenAggregation = new AggregationNode( aggregation.getSourceLocation(), @@ -156,7 +158,7 @@ public Result apply(AggregationNode aggregation, Captures captures, Context cont aggregation.getAggregationId()); JoinNode rewrittenJoin; - if (join.getType() == JoinNode.Type.LEFT) { + if (join.getType() == JoinType.LEFT) { rewrittenJoin = new JoinNode( join.getSourceLocation(), join.getId(), @@ -203,9 +205,9 @@ public Result apply(AggregationNode aggregation, Captures captures, Context cont private static PlanNode getInnerTable(JoinNode join) { - checkState(join.getType() == JoinNode.Type.LEFT || join.getType() == JoinNode.Type.RIGHT, "expected LEFT or RIGHT JOIN"); + checkState(join.getType() == JoinType.LEFT || join.getType() == JoinType.RIGHT, "expected LEFT or RIGHT JOIN"); PlanNode innerNode; - if (join.getType().equals(JoinNode.Type.LEFT)) { + if (join.getType().equals(JoinType.LEFT)) { innerNode = join.getRight(); } else { @@ -216,9 +218,9 @@ private static PlanNode getInnerTable(JoinNode join) private static PlanNode getOuterTable(JoinNode join) { - checkState(join.getType() == JoinNode.Type.LEFT || join.getType() == JoinNode.Type.RIGHT, "expected LEFT or RIGHT JOIN"); + checkState(join.getType() == JoinType.LEFT || join.getType() == JoinType.RIGHT, "expected LEFT or RIGHT JOIN"); PlanNode outerNode; - if (join.getType().equals(JoinNode.Type.LEFT)) { + if (join.getType().equals(JoinType.LEFT)) { outerNode = join.getLeft(); } else { @@ -282,7 +284,7 @@ else if (!functionAndTypeManager.getFunctionMetadata(functionHandle).isCalledOnN finalJoinNode = new JoinNode( outerJoin.getSourceLocation(), idAllocator.getNextId(), - JoinNode.Type.INNER, + JoinType.INNER, outerJoin, aggregationOverNull, ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownFilterExpressionEvaluationThroughCrossJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownFilterExpressionEvaluationThroughCrossJoin.java index efb92939c6c84..4caec2cf5e560 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownFilterExpressionEvaluationThroughCrossJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownFilterExpressionEvaluationThroughCrossJoin.java @@ -20,6 +20,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; @@ -81,7 +82,7 @@ public class PushDownFilterExpressionEvaluationThroughCrossJoin private static final Capture CHILD = newCapture(); private static final Pattern PATTERN = filter() - .with(source().matching(join().matching(x -> x.getCriteria().isEmpty() && x.getType().equals(JoinNode.Type.INNER)).capturedAs(CHILD))); + .with(source().matching(join().matching(x -> x.getCriteria().isEmpty() && x.getType().equals(JoinType.INNER)).capturedAs(CHILD))); private final FunctionAndTypeManager functionAndTypeManager; private final RowExpressionDeterminismEvaluator determinismEvaluator; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java index 3ea81bb2c49fa..917ba1e679dfe 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java @@ -26,10 +26,10 @@ import static com.facebook.presto.SystemSessionProperties.isPushLimitThroughOuterJoin; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.spi.plan.LimitNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.extractCardinality; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.planner.plan.Patterns.Join.type; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.facebook.presto.sql.planner.plan.Patterns.limit; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java index 025ecd18b10bd..442fd8ebd63b5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java @@ -18,6 +18,8 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; @@ -85,7 +87,7 @@ public Result apply(AggregationNode aggregationNode, Captures captures, Context { JoinNode joinNode = captures.get(JOIN_NODE); - if (joinNode.getType() != JoinNode.Type.INNER) { + if (joinNode.getType() != JoinType.INNER) { return Result.empty(); } @@ -131,8 +133,8 @@ private PlanNode pushPartialToRightChild(AggregationNode node, JoinNode child, C private Set getJoinRequiredVariables(JoinNode node) { return Streams.concat( - node.getCriteria().stream().map(JoinNode.EquiJoinClause::getLeft), - node.getCriteria().stream().map(JoinNode.EquiJoinClause::getRight), + node.getCriteria().stream().map(EquiJoinClause::getLeft), + node.getCriteria().stream().map(EquiJoinClause::getRight), node.getFilter().map(expression -> VariablesExtractor.extractUnique(expression)).orElse(ImmutableSet.of()).stream(), node.getLeftHashVariable().map(ImmutableSet::of).orElse(ImmutableSet.of()).stream(), node.getRightHashVariable().map(ImmutableSet::of).orElse(ImmutableSet.of()).stream()) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantCastToVarcharInJoinClause.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantCastToVarcharInJoinClause.java index 8d7b478215fc7..18fd658c31bea 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantCastToVarcharInJoinClause.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantCastToVarcharInJoinClause.java @@ -19,6 +19,7 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; @@ -114,11 +115,11 @@ public Result apply(JoinNode node, Captures captures, Context context) ProjectNode leftProject = (ProjectNode) leftInput; ProjectNode rightProject = (ProjectNode) rightInput; - ImmutableList.Builder joinClauseBuilder = ImmutableList.builder(); + ImmutableList.Builder joinClauseBuilder = ImmutableList.builder(); ImmutableMap.Builder newLeftAssignmentsBuilder = ImmutableMap.builder(); ImmutableMap.Builder newRightAssignmentsBuilder = ImmutableMap.builder(); boolean isChanged = false; - for (JoinNode.EquiJoinClause equiJoinClause : node.getCriteria()) { + for (EquiJoinClause equiJoinClause : node.getCriteria()) { RowExpression leftProjectAssignment = leftProject.getAssignments().getMap().get(equiJoinClause.getLeft()); RowExpression rightProjectAssignment = rightProject.getAssignments().getMap().get(equiJoinClause.getRight()); if (!isSupportedCast(leftProjectAssignment) || !isSupportedCast(rightProjectAssignment)) { @@ -140,7 +141,7 @@ public Result apply(JoinNode node, Captures captures, Context context) VariableReferenceExpression newRight = context.getVariableAllocator().newVariable(rightAssignment); newRightAssignmentsBuilder.put(newRight, rightAssignment); - joinClauseBuilder.add(new JoinNode.EquiJoinClause(newLeft, newRight)); + joinClauseBuilder.add(new EquiJoinClause(newLeft, newRight)); isChanged = true; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java index 0acfb7df425e6..6b1709bffa9f3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java @@ -25,6 +25,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; @@ -40,8 +41,6 @@ import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.DistributionType; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.google.common.annotations.VisibleForTesting; @@ -74,6 +73,9 @@ import static com.facebook.presto.expressions.LogicalRowExpressions.and; import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; import static com.facebook.presto.sql.planner.EqualityInference.createEqualityInference; @@ -86,9 +88,6 @@ import static com.facebook.presto.sql.planner.optimizations.JoinNodeUtils.toRowExpression; import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.getNonIdentityAssignments; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -569,12 +568,12 @@ private List getPossibleJoinNodes(JoinNode joinNode, Join } } - private List getPossibleJoinNodes(JoinNode joinNode, DistributionType distributionType) + private List getPossibleJoinNodes(JoinNode joinNode, com.facebook.presto.spi.plan.JoinDistributionType distributionType) { return getPossibleJoinNodes(joinNode, distributionType, (node) -> true); } - private List getPossibleJoinNodes(JoinNode joinNode, DistributionType distributionType, Predicate isAllowed) + private List getPossibleJoinNodes(JoinNode joinNode, com.facebook.presto.spi.plan.JoinDistributionType distributionType, Predicate isAllowed) { List nodes = ImmutableList.of( joinNode.withDistributionType(distributionType), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RuntimeReorderJoinSides.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RuntimeReorderJoinSides.java index dd90dab4a6c7d..34ff2a8216adc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RuntimeReorderJoinSides.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RuntimeReorderJoinSides.java @@ -27,12 +27,12 @@ import java.util.Optional; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.createRuntimeSwappedJoinNode; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java index 615d4256cda63..e22d901e2b212 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; @@ -260,7 +261,7 @@ private static JoinNode leftOuterJoin(PlanNodeIdAllocator idAllocator, AssignUni return new JoinNode( probeSide.getSourceLocation(), idAllocator.getNextId(), - JoinNode.Type.LEFT, + JoinType.LEFT, probeSide, buildSide, ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java index 76c21eed2b7cc..0250daed9e42d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToLeftEarlyOutJoin.java @@ -19,6 +19,7 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.LogicalProperties; import com.facebook.presto.spi.plan.ProjectNode; @@ -37,10 +38,9 @@ import static com.facebook.presto.SystemSessionProperties.isInPredicatesAsInnerJoinsEnabled; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; -import static com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.Patterns.Join.type; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static com.facebook.presto.sql.planner.plan.Patterns.join; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java index d1d4cc12a4c0c..158632a228fa9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformDistinctInnerJoinToRightEarlyOutJoin.java @@ -39,8 +39,8 @@ import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.Patterns.Join.type; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static com.facebook.presto.sql.planner.plan.Patterns.join; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java index ef4abc92d84ea..7acc9ce23c359 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java @@ -18,6 +18,8 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.InSubqueryExpression; @@ -28,7 +30,6 @@ import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -127,7 +128,7 @@ public Result apply(ApplyNode applyNode, Captures captures, Context context) JoinNode innerJoin = new JoinNode( applyNode.getSourceLocation(), context.getIdAllocator().getNextId(), - JoinNode.Type.INNER, + JoinType.INNER, leftInput, applyNode.getSubquery(), ImmutableList.of(new EquiJoinClause( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedLateralToJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedLateralToJoin.java index 62c3d45fba5b0..e2439fd2fe004 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedLateralToJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformUncorrelatedLateralToJoin.java @@ -15,6 +15,7 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.JoinNode; @@ -46,7 +47,7 @@ public Result apply(LateralJoinNode lateralJoinNode, Captures captures, Context return Result.ofPlanNode(new JoinNode( lateralJoinNode.getSourceLocation(), context.getIdAllocator().getNextId(), - JoinNode.Type.INNER, + JoinType.INNER, lateralJoinNode.getInput(), lateralJoinNode.getSubquery(), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java index 868ecbf3b6373..37543bc2233e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java @@ -29,7 +29,9 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.OutputNode; @@ -813,15 +815,15 @@ private Function createDirectTranslator(SetMultimap inputToOutpu public PlanWithProperties visitJoin(JoinNode node, PreferredProperties preferredProperties) { List leftVariables = node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getLeft) + .map(EquiJoinClause::getLeft) .collect(toImmutableList()); List rightVariables = node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .collect(toImmutableList()); - JoinNode.DistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); + JoinDistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); - if (distributionType == JoinNode.DistributionType.REPLICATED) { + if (distributionType == JoinDistributionType.REPLICATED) { PlanWithProperties left = accept(node.getLeft(), PreferredProperties.any()); // use partitioned join if probe side is naturally partitioned on join symbols (e.g: because of aggregation) @@ -914,7 +916,7 @@ private PlanWithProperties planPartitionedJoin(JoinNode node, List buildHashVariables = node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .collect(toImmutableList()); StreamPreferredProperties buildPreference; if (getTaskConcurrency(session) > 1) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java index d5144ec97a4a9..6ec23173110d7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java @@ -14,11 +14,13 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPlanOptimizer; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ConnectorJoinNode; import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; @@ -36,6 +38,8 @@ import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -146,6 +150,9 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider containsAll(ImmutableSet.copyOf(newNode.getOutputVariables()), node.getOutputVariables()), "the connector optimizer from %s returns a node that does not cover all output before optimization", connectorId); + + newNode = SimplePlanRewriter.rewriteWith(new ConnectorToInternalJoinRewriter(), newNode); + updates.put(node, newNode); } } @@ -292,4 +299,26 @@ private static boolean containsAll(Set container, Collection test) } return true; } + + private static class ConnectorToInternalJoinRewriter + extends SimplePlanRewriter + { + @Override + public PlanNode visitConnectorJoinNode(ConnectorJoinNode node, RewriteContext context) + { + return new JoinNode(node.getSourceLocation(), + node.getId(), + node.getStatsEquivalentPlanNode(), + node.getType(), + context.rewrite(node.getSources().get(0)), + context.rewrite(node.getSources().get(1)), + ImmutableList.copyOf(node.getCriteria()), + node.getOutputVariables(), + node.getFilters().isEmpty() ? Optional.empty() : Optional.of(LogicalRowExpressions.and(node.getFilters())), + Optional.empty(), + Optional.empty(), + node.getDistributionType(), + ImmutableMap.of()); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java index e62d20849f97a..a8a44479c2d6a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -69,6 +70,9 @@ import static com.facebook.presto.SystemSessionProperties.skipHashGenerationForJoinWithTableScanInput; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.spi.plan.ProjectNode.Locality.REMOTE; import static com.facebook.presto.sql.planner.PlannerUtils.HASH_CODE; @@ -77,9 +81,6 @@ import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.google.common.base.MoreObjects.toStringHelper; @@ -343,7 +344,7 @@ private boolean skipHashComputeForJoinInput(PlanNode node, Optional clauses = node.getCriteria(); + List clauses = node.getCriteria(); if (clauses.isEmpty()) { // join does not pass through preferred hash variables since they take more memory and since // the join node filters, may take more compute @@ -357,14 +358,14 @@ public PlanWithProperties visitJoin(JoinNode node, HashComputationSet parentPref // join does not pass through preferred hash variables since they take more memory and since // the join node filters, may take more compute - Optional leftHashComputation = computeHash(Lists.transform(clauses, JoinNode.EquiJoinClause::getLeft), functionAndTypeManager); + Optional leftHashComputation = computeHash(Lists.transform(clauses, EquiJoinClause::getLeft), functionAndTypeManager); if (skipHashGenerationForJoinWithTableScanInput(session) && skipHashComputeForJoinInput(node.getLeft(), leftHashComputation, parentPreference)) { leftHashComputation = Optional.empty(); } PlanWithProperties left = planAndEnforce(node.getLeft(), new HashComputationSet(leftHashComputation), true, new HashComputationSet(leftHashComputation)); Optional leftHashVariable = leftHashComputation.isPresent() ? Optional.of(left.getRequiredHashVariable(leftHashComputation.get())) : Optional.empty(); - Optional rightHashComputation = computeHash(Lists.transform(clauses, JoinNode.EquiJoinClause::getRight), functionAndTypeManager); + Optional rightHashComputation = computeHash(Lists.transform(clauses, EquiJoinClause::getRight), functionAndTypeManager); if (skipHashGenerationForJoinWithTableScanInput(session) && skipHashComputeForJoinInput(node.getRight(), rightHashComputation, parentPreference)) { rightHashComputation = Optional.empty(); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java index e8abb7728b57f..48272691adbdb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -117,8 +118,8 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) PlanNode rightRewritten = context.rewrite(node.getRight()); if (!node.getCriteria().isEmpty()) { // Index join only possible with JOIN criteria - List leftJoinVariables = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getLeft); - List rightJoinVariables = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight); + List leftJoinVariables = Lists.transform(node.getCriteria(), EquiJoinClause::getLeft); + List rightJoinVariables = Lists.transform(node.getCriteria(), EquiJoinClause::getRight); Optional leftIndexCandidate = IndexSourceRewriter.rewriteWithIndex( leftRewritten, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinNodeUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinNodeUtils.java index c491a9a7d64cd..6941398ef68c1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinNodeUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinNodeUtils.java @@ -14,20 +14,20 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.common.function.OperatorType; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.Join; import com.google.common.collect.ImmutableList; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.createSymbolReference; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.EQUAL; @@ -49,7 +49,7 @@ public static RowExpression toRowExpression(EquiJoinClause clause, FunctionResol ImmutableList.of(clause.getLeft(), clause.getRight())); } - public static JoinNode.Type typeConvert(Join.Type joinType) + public static JoinType typeConvert(Join.Type joinType) { // Omit SEMI join types because they must be inferred by the planner and not part of the SQL parse tree switch (joinType) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java index e427ad753fdc6..ae9b0bb3905ce 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -227,7 +228,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) // Find the best equijoin clause so we sample both sides the same way optimally // First see if there is a int/bigint key - Optional equiJoinClause = node.getCriteria().stream() + Optional equiJoinClause = node.getCriteria().stream() .filter(x -> TypeUtils.isIntegralType(x.getLeft().getType().getTypeSignature(), functionAndTypeManager)) .findFirst(); if (!equiJoinClause.isPresent()) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java index 83951e5674271..ba8edbd8a610b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MergeJoinForSortedInputOptimizer.java @@ -17,6 +17,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -31,7 +32,7 @@ import static com.facebook.presto.SystemSessionProperties.isGroupedExecutionEnabled; import static com.facebook.presto.SystemSessionProperties.preferMergeJoinForSortedInputs; import static com.facebook.presto.common.block.SortOrder.ASC_NULLS_FIRST; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; @@ -143,9 +144,9 @@ private boolean meetsDataRequirement(PlanNode left, PlanNode right, JoinNode nod StreamPropertyDerivations.StreamProperties leftProperties = StreamPropertyDerivations.derivePropertiesRecursively(left, metadata, session, types, parser); StreamPropertyDerivations.StreamProperties rightProperties = StreamPropertyDerivations.derivePropertiesRecursively(right, metadata, session, types, parser); - List leftJoinColumns = node.getCriteria().stream().map(JoinNode.EquiJoinClause::getLeft).collect(toImmutableList()); + List leftJoinColumns = node.getCriteria().stream().map(EquiJoinClause::getLeft).collect(toImmutableList()); List rightJoinColumns = node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .collect(toImmutableList()); // Check if both the left side and right side's partitioning columns (bucketed-by columns [B]) are a subset of join columns [J] diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java index e82ce42a82cea..74a8da8e74712 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PayloadJoinOptimizer.java @@ -23,7 +23,9 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; @@ -56,6 +58,7 @@ import static com.facebook.presto.common.type.TypeUtils.isNumericType; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; import static com.facebook.presto.sql.planner.PlannerUtils.addProjections; import static com.facebook.presto.sql.planner.PlannerUtils.clonePlanNode; @@ -64,7 +67,6 @@ import static com.facebook.presto.sql.planner.PlannerUtils.isScanFilterProject; import static com.facebook.presto.sql.planner.PlannerUtils.restrictOutput; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.specialForm; import static com.google.common.base.Preconditions.checkArgument; @@ -267,7 +269,7 @@ private PlanNode defaultRewriteJoinChild(PlanNode child, RewriteContext leftColumns, Set joinKeys) + private boolean needsRewrite(JoinType joinType, ImmutableSet leftColumns, Set joinKeys) { return joinType == LEFT && supportedJoinKeyTypes(joinKeys) && leftColumns.stream().anyMatch(var -> !joinKeys.contains(var)); } @@ -443,7 +445,7 @@ private PlanNode transformJoin(JoinNode keysNode, JoinContext context) return new JoinNode( keysNode.getSourceLocation(), planNodeIdAllocator.getNextId(), - JoinNode.Type.LEFT, + JoinType.LEFT, payloadPlanNode, projectNode, ImmutableList.of(), @@ -488,7 +490,7 @@ private RowExpression coalesceToZero(RowExpression var) return coalesce(ImmutableList.of(var, zero)); } - private Set extractJoinKeys(Optional filter, List criteria) + private Set extractJoinKeys(Optional filter, List criteria) { ImmutableSet.Builder builder = ImmutableSet.builder(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java index d2e9d1851ef87..35cdb5f3091f8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java @@ -26,7 +26,10 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -96,6 +99,12 @@ import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.spi.plan.ProjectNode.Locality; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.spi.plan.ProjectNode.Locality.REMOTE; @@ -103,12 +112,6 @@ import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.VariablesExtractor.extractUnique; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.constantNull; @@ -515,7 +518,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) Locality leftLocality = LOCAL; Locality rightLocality = LOCAL; // Create new projections for the new join clauses - List equiJoinClauses = new ArrayList<>(); + List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); for (RowExpression conjunct : extractConjuncts(newJoinPredicate)) { if (joinEqualityExpression(node.getLeft().getOutputVariables()).test(conjunct)) { @@ -539,7 +542,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) } } - equiJoinClauses.add(new JoinNode.EquiJoinClause(leftVariable, rightVariable)); + equiJoinClauses.add(new EquiJoinClause(leftVariable, rightVariable)); } else { joinFilterBuilder.add(conjunct); @@ -604,7 +607,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) // if the distribution type is already set, make sure that changes from PredicatePushDown // don't make the join node invalid. - Optional distributionType = node.getDistributionType(); + Optional distributionType = node.getDistributionType(); if (node.getDistributionType().isPresent()) { if (node.getType().mustPartition()) { distributionType = Optional.of(PARTITIONED); @@ -676,7 +679,7 @@ private PlanNode wrapInProjectIfNeeded(PlanNode childNode, Assignments assignmen private static DynamicFiltersResult createDynamicFilters( JoinNode node, - List equiJoinClauses, + List equiJoinClauses, List joinFilter, PlanNodeIdAllocator idAllocator, FunctionAndTypeManager functionAndTypeManager) @@ -710,7 +713,7 @@ private static DynamicFiltersResult createDynamicFilters( private static List getDynamicFilterClauses( JoinNode node, - List equiJoinClauses, + List equiJoinClauses, List joinFilter, FunctionAndTypeManager functionAndTypeManager) { @@ -719,7 +722,7 @@ private static List getDynamicFilterClauses( // instead of separate ApplyDynamicFilters rule we derive dynamic filters within PredicatePushdown itself. // Even if equiJoinClauses.equals(node.getCriteria), current dynamic filters may not match equiJoinClauses ImmutableList.Builder clausesBuilder = ImmutableList.builder(); - for (JoinNode.EquiJoinClause clause : equiJoinClauses) { + for (EquiJoinClause clause : equiJoinClauses) { VariableReferenceExpression probeSymbol = clause.getLeft(); VariableReferenceExpression buildSymbol = clause.getRight(); clausesBuilder.add(call( @@ -1319,14 +1322,14 @@ private RowExpression getPostJoinPredicate() private RowExpression extractJoinPredicate(JoinNode joinNode) { ImmutableList.Builder builder = ImmutableList.builder(); - for (JoinNode.EquiJoinClause equiJoinClause : joinNode.getCriteria()) { + for (EquiJoinClause equiJoinClause : joinNode.getCriteria()) { builder.add(toRowExpression(equiJoinClause)); } joinNode.getFilter().ifPresent(builder::add); return logicalRowExpressions.combineConjuncts(builder.build()); } - private RowExpression toRowExpression(JoinNode.EquiJoinClause equiJoinClause) + private RowExpression toRowExpression(EquiJoinClause equiJoinClause) { return buildEqualsExpression(functionAndTypeManager, equiJoinClause.getLeft(), equiJoinClause.getRight()); } @@ -1335,11 +1338,11 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, RowExpression inh { checkArgument(EnumSet.of(INNER, RIGHT, LEFT, FULL).contains(node.getType()), "Unsupported join type: %s", node.getType()); - if (node.getType() == JoinNode.Type.INNER) { + if (node.getType() == JoinType.INNER) { return node; } - if (node.getType() == JoinNode.Type.FULL) { + if (node.getType() == JoinType.FULL) { boolean canConvertToLeftJoin = canConvertOuterToInner(node.getLeft().getOutputVariables(), inheritedPredicate); boolean canConvertToRightJoin = canConvertOuterToInner(node.getRight().getOutputVariables(), inheritedPredicate); if (!canConvertToLeftJoin && !canConvertToRightJoin) { @@ -1377,14 +1380,14 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, RowExpression inh } } - if (node.getType() == JoinNode.Type.LEFT && !canConvertOuterToInner(node.getRight().getOutputVariables(), inheritedPredicate) || - node.getType() == JoinNode.Type.RIGHT && !canConvertOuterToInner(node.getLeft().getOutputVariables(), inheritedPredicate)) { + if (node.getType() == JoinType.LEFT && !canConvertOuterToInner(node.getRight().getOutputVariables(), inheritedPredicate) || + node.getType() == JoinType.RIGHT && !canConvertOuterToInner(node.getLeft().getOutputVariables(), inheritedPredicate)) { return node; } return new JoinNode( node.getSourceLocation(), node.getId(), - JoinNode.Type.INNER, + JoinType.INNER, node.getLeft(), node.getRight(), node.getCriteria(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java index 4d2f808f09543..6e7c72947ff0b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java @@ -49,6 +49,7 @@ import static com.facebook.presto.common.function.OperatorType.EQUAL; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IF; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; @@ -62,7 +63,6 @@ import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.isAllLowCardinalityGroupByKeys; import static com.facebook.presto.sql.planner.optimizations.JoinNodeUtils.typeConvert; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.specialForm; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java index 460c5631c9bc2..b2058d40f7ddc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java @@ -25,7 +25,9 @@ import com.facebook.presto.spi.SortingProperty; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.OrderingScheme; @@ -864,7 +866,7 @@ private static Optional> translateToNonConstan } } - static boolean spillPossible(Session session, JoinNode.Type joinType) + static boolean spillPossible(Session session, JoinType joinType) { if (!isSpillEnabled(session) || !isJoinSpillingEnabled(session)) { return false; @@ -894,7 +896,7 @@ public static Optional filterIfMissing(Collection filterOrRewrite(Collection columns, List equalities, VariableReferenceExpression column) + public static Optional filterOrRewrite(Collection columns, List equalities, VariableReferenceExpression column) { // symbol is exposed directly, so no translation needed if (columns.contains(column)) { @@ -903,7 +905,7 @@ public static Optional filterOrRewrite(Collection leftInputsBuilder = ImmutableSet.builder(); - leftInputsBuilder.addAll(context.get()).addAll(Iterables.transform(node.getCriteria(), JoinNode.EquiJoinClause::getLeft)); + leftInputsBuilder.addAll(context.get()).addAll(Iterables.transform(node.getCriteria(), EquiJoinClause::getLeft)); if (node.getLeftHashVariable().isPresent()) { leftInputsBuilder.add(node.getLeftHashVariable().get()); } @@ -222,7 +223,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext leftInputs = leftInputsBuilder.build(); ImmutableSet.Builder rightInputsBuilder = ImmutableSet.builder(); - rightInputsBuilder.addAll(context.get()).addAll(Iterables.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight)); + rightInputsBuilder.addAll(context.get()).addAll(Iterables.transform(node.getCriteria(), EquiJoinClause::getRight)); if (node.getRightHashVariable().isPresent()) { rightInputsBuilder.add(node.getRightHashVariable().get()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index 79b13554dc644..22c761c79970d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -36,6 +36,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.OrderingScheme; @@ -259,10 +260,10 @@ public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext conte public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getLeft) + .map(EquiJoinClause::getLeft) .forEach(context.get().variables::add); node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .forEach(context.get().variables::add); node.getFilter() diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java index 37c53f118f708..6791135c1d0be 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RandomizeNullKeyInOuterJoin.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; @@ -54,6 +55,10 @@ import static com.facebook.presto.common.type.DateType.DATE; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.metadata.CastType.CAST; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; @@ -63,10 +68,6 @@ import static com.facebook.presto.sql.analyzer.FeaturesConfig.RandomizeOuterJoinNullKeyStrategy.DISABLED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.RandomizeOuterJoinNullKeyStrategy.KEY_FROM_OUTER_JOIN; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.specialForm; @@ -288,7 +289,7 @@ public PlanNode visitJoin(JoinNode joinNode, RewriteContext candidateEquiJoinClauses = joinNode.getCriteria().stream() + List candidateEquiJoinClauses = joinNode.getCriteria().stream() .filter(x -> isSupportedType(x.getLeft()) && isSupportedType(x.getRight())) .filter(x -> enabledByCostModel || strategy.equals(ALWAYS) || enabledForJoinKeyFromOuterJoin(context.get(), x)) .collect(toImmutableList()); @@ -313,14 +314,14 @@ public PlanNode visitJoin(JoinNode joinNode, RewriteContext rightKeyRandomVariableMap = generateRandomKeyMap(rightJoinKeys, RIGHT_PREFIX); - ImmutableList.Builder joinClauseBuilder = ImmutableList.builder(); + ImmutableList.Builder joinClauseBuilder = ImmutableList.builder(); // Rewrite supported join clauses - List rewrittenJoinClauses = candidateEquiJoinClauses.stream() - .map(x -> new JoinNode.EquiJoinClause(keyToRandomKeyMap.get(LEFT_PREFIX).get(x.getLeft()), keyToRandomKeyMap.get(RIGHT_PREFIX).get(x.getRight()))) + List rewrittenJoinClauses = candidateEquiJoinClauses.stream() + .map(x -> new EquiJoinClause(keyToRandomKeyMap.get(LEFT_PREFIX).get(x.getLeft()), keyToRandomKeyMap.get(RIGHT_PREFIX).get(x.getRight()))) .collect(toImmutableList()); joinClauseBuilder.addAll(rewrittenJoinClauses); // Add the join clauses which are not supported back - List unchangedJoinClauses = joinNode.getCriteria().stream() + List unchangedJoinClauses = joinNode.getCriteria().stream() .filter(x -> !candidateEquiJoinClauses.contains(x)) .collect(toImmutableList()); joinClauseBuilder.addAll(unchangedJoinClauses); @@ -333,9 +334,9 @@ public PlanNode visitJoin(JoinNode joinNode, RewriteContext x.getRight().getType() instanceof VarcharType).map(x -> x.getRight()).distinct().collect(toImmutableMap(identity(), x -> specialForm(IS_NULL, BOOLEAN, x))); Map rightIsNullCheckAssignment = rightIsNullCheckExpression.values().stream().collect(toImmutableMap(identity(), x -> planVariableAllocator.newVariable(x))); - List isNullCheck = candidateEquiJoinClauses.stream() + List isNullCheck = candidateEquiJoinClauses.stream() .filter(x -> x.getLeft().getType() instanceof VarcharType && x.getRight().getType() instanceof VarcharType) - .map(x -> new JoinNode.EquiJoinClause(leftIsNullCheckAssignment.get(leftIsNullCheckExpression.get(x.getLeft())), rightIsNullCheckAssignment.get(rightIsNullCheckExpression.get(x.getRight())))) + .map(x -> new EquiJoinClause(leftIsNullCheckAssignment.get(leftIsNullCheckExpression.get(x.getLeft())), rightIsNullCheckAssignment.get(rightIsNullCheckExpression.get(x.getRight())))) .collect(toImmutableList()); joinClauseBuilder.addAll(isNullCheck); @@ -417,7 +418,7 @@ private boolean isAlreadyRandomized(PlanNode source, VariableReferenceExpression return keyToRandomKeyMap.containsKey(prefix) && keyToRandomKeyMap.get(prefix).containsKey(joinKey) && source.getOutputVariables().contains(keyToRandomKeyMap.get(prefix).get(joinKey)); } - private boolean enabledForJoinKeyFromOuterJoin(Set variablesFromOuterJoin, JoinNode.EquiJoinClause joinClause) + private boolean enabledForJoinKeyFromOuterJoin(Set variablesFromOuterJoin, EquiJoinClause joinClause) { return strategy.equals(KEY_FROM_OUTER_JOIN) && (variablesFromOuterJoin.contains(joinClause.getLeft()) || variablesFromOuterJoin.contains(joinClause.getRight())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplaceConstantVariableReferencesWithConstants.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplaceConstantVariableReferencesWithConstants.java index 77fea39659c42..34e0c761930fa 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplaceConstantVariableReferencesWithConstants.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplaceConstantVariableReferencesWithConstants.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -252,11 +253,11 @@ public PlanNodeWithConstant visitJoin(JoinNode node, Void context) ImmutableMap.Builder outputConstantMap = ImmutableMap.builder(); // Output from inner side of outer joins can be NULL when no match, hence will not keep the constant constraint - if (node.getType().equals(JoinNode.Type.LEFT) || node.getType().equals(JoinNode.Type.INNER)) { + if (node.getType().equals(JoinType.LEFT) || node.getType().equals(JoinType.INNER)) { outputConstantMap.putAll(rewrittenLeft.getConstantExpressionMap()); } - if (node.getType().equals(JoinNode.Type.RIGHT) || node.getType().equals(JoinNode.Type.INNER)) { + if (node.getType().equals(JoinType.RIGHT) || node.getType().equals(JoinType.INNER)) { outputConstantMap.putAll(rewrittenRight.getConstantExpressionMap()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java index 4390ec11b1884..3aacc644a555b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; @@ -117,7 +118,7 @@ private PlanNode rewriteScalarAggregation( JoinNode leftOuterJoin = new JoinNode( scalarAggregation.getSourceLocation(), idAllocator.getNextId(), - JoinNode.Type.LEFT, + JoinType.LEFT, inputWithUniqueColumns, scalarAggregationSource, ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ShardJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ShardJoins.java index ce31a01a9f212..ab89d1234ffac 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ShardJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ShardJoins.java @@ -20,6 +20,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.RowExpression; @@ -41,12 +42,12 @@ import static com.facebook.presto.SystemSessionProperties.getJoinShardCount; import static com.facebook.presto.SystemSessionProperties.getShardedJoinStrategy; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.analyzer.FeaturesConfig.ShardedJoinStrategy.ALWAYS; import static com.facebook.presto.sql.analyzer.FeaturesConfig.ShardedJoinStrategy.COST_BASED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.ShardedJoinStrategy.DISABLED; import static com.facebook.presto.sql.planner.PlannerUtils.isBroadcastJoin; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.google.common.base.Preconditions.checkState; @@ -155,8 +156,8 @@ public PlanNode visitJoin(JoinNode joinNode, RewriteContext joinCriteria = new ArrayList<>(); + EquiJoinClause shardEquality = new EquiJoinClause(leftShardVariable, rightShardVariable); + List joinCriteria = new ArrayList<>(); joinCriteria.addAll(joinNode.getCriteria()); joinCriteria.add(shardEquality); PlanNode result = new JoinNode( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index d48839efb52f8..5c72e3b957c4a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; @@ -91,9 +92,9 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.getNodeLocation; import static com.facebook.presto.sql.planner.optimizations.ApplyNodeUtil.verifySubquerySupported; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.isNull; import static com.google.common.base.Preconditions.checkState; @@ -568,7 +569,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) PlanNode left = context.rewrite(node.getLeft()); PlanNode right = context.rewrite(node.getRight()); - List canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); + List canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); Optional canonicalFilter = node.getFilter().map(this::canonicalize); Optional canonicalLeftHashVariable = canonicalize(node.getLeftHashVariable()); Optional canonicalRightHashVariable = canonicalize(node.getRightHashVariable()); @@ -809,11 +810,11 @@ private Set canonicalize(Set canonicalizeJoinCriteria(List criteria) + private List canonicalizeJoinCriteria(List criteria) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (JoinNode.EquiJoinClause clause : criteria) { - builder.add(new JoinNode.EquiJoinClause(canonicalize(clause.getLeft()), canonicalize(clause.getRight()))); + ImmutableList.Builder builder = ImmutableList.builder(); + for (EquiJoinClause clause : criteria) { + builder.add(new EquiJoinClause(canonicalize(clause.getLeft()), canonicalize(clause.getRight()))); } return builder.build(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java index 73b263b489d60..86a53a889b545 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner.optimizations.joins; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; @@ -34,7 +35,7 @@ import java.util.Map; import java.util.Optional; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.relational.ProjectNodeUtils.isIdentity; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -182,7 +183,7 @@ public String toString() return builder.toString(); } - private JoinGraph joinWith(JoinGraph other, List joinClauses, Context context, PlanNodeId newRoot) + private JoinGraph joinWith(JoinGraph other, List joinClauses, Context context, PlanNodeId newRoot) { for (PlanNode node : other.nodes) { checkState(!edges.containsKey(node.getId()), format("Node [%s] appeared in two JoinGraphs", node)); @@ -202,7 +203,7 @@ private JoinGraph joinWith(JoinGraph other, List joinCl .addAll(other.filters) .build(); - for (JoinNode.EquiJoinClause edge : joinClauses) { + for (EquiJoinClause edge : joinClauses) { VariableReferenceExpression leftVariable = edge.getLeft(); VariableReferenceExpression rightVariable = edge.getRight(); checkState(context.containsVariable(leftVariable)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/JoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/JoinNode.java index cb8b8731bafee..a3f2c455585c4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/JoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/JoinNode.java @@ -15,6 +15,9 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LogicalProperties; import com.facebook.presto.spi.plan.LogicalPropertiesProvider; import com.facebook.presto.spi.plan.PlanNode; @@ -35,18 +38,17 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.SortExpressionExtractor.extractSortExpression; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.planner.sanity.ValidateDependenciesChecker.checkLeftOutputVariablesBeforeRight; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -57,7 +59,7 @@ public class JoinNode extends AbstractJoinNode { - private final Type type; + private final JoinType type; private final PlanNode left; private final PlanNode right; private final List criteria; @@ -65,14 +67,14 @@ public class JoinNode private final Optional filter; private final Optional leftHashVariable; private final Optional rightHashVariable; - private final Optional distributionType; + private final Optional distributionType; private final Map dynamicFilters; @JsonCreator public JoinNode( Optional sourceLocation, @JsonProperty("id") PlanNodeId id, - @JsonProperty("type") Type type, + @JsonProperty("type") JoinType type, @JsonProperty("left") PlanNode left, @JsonProperty("right") PlanNode right, @JsonProperty("criteria") List criteria, @@ -80,7 +82,7 @@ public JoinNode( @JsonProperty("filter") Optional filter, @JsonProperty("leftHashVariable") Optional leftHashVariable, @JsonProperty("rightHashVariable") Optional rightHashVariable, - @JsonProperty("distributionType") Optional distributionType, + @JsonProperty("distributionType") Optional distributionType, @JsonProperty("dynamicFilters") Map dynamicFilters) { this(sourceLocation, id, Optional.empty(), type, left, right, criteria, outputVariables, filter, leftHashVariable, rightHashVariable, distributionType, dynamicFilters); @@ -90,7 +92,7 @@ public JoinNode( Optional sourceLocation, PlanNodeId id, Optional statsEquivalentPlanNode, - Type type, + JoinType type, PlanNode left, PlanNode right, List criteria, @@ -98,7 +100,7 @@ public JoinNode( Optional filter, Optional leftHashVariable, Optional rightHashVariable, - Optional distributionType, + Optional distributionType, Map dynamicFilters) { super(sourceLocation, id, statsEquivalentPlanNode); @@ -179,7 +181,7 @@ public JoinNode flipChildren() } @VisibleForTesting - public static Type flipType(Type type) + public static JoinType flipType(JoinType type) { switch (type) { case INNER: @@ -216,46 +218,8 @@ private static List flipOutputVariables(List criteria) - { - // There is nothing to partition on - return criteria.isEmpty() && (this == INNER || this == LEFT); - } - } - @JsonProperty - public Type getType() + public JoinType getType() { return type; } @@ -335,7 +299,7 @@ public List getOutputVariables() } @JsonProperty - public Optional getDistributionType() + public Optional getDistributionType() { return distributionType; } @@ -366,7 +330,7 @@ public PlanNode assignStatsEquivalentPlanNode(Optional statsEquivalent return new JoinNode(getSourceLocation(), getId(), statsEquivalentPlanNode, type, left, right, criteria, outputVariables, filter, leftHashVariable, rightHashVariable, distributionType, dynamicFilters); } - public JoinNode withDistributionType(DistributionType distributionType) + public JoinNode withDistributionType(JoinDistributionType distributionType) { return new JoinNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), type, left, right, criteria, outputVariables, filter, leftHashVariable, rightHashVariable, Optional.of(distributionType), dynamicFilters); } @@ -375,63 +339,4 @@ public boolean isCrossJoin() { return criteria.isEmpty() && !filter.isPresent() && type == INNER; } - - public static class EquiJoinClause - { - private final VariableReferenceExpression left; - private final VariableReferenceExpression right; - - @JsonCreator - public EquiJoinClause(@JsonProperty("left") VariableReferenceExpression left, @JsonProperty("right") VariableReferenceExpression right) - { - this.left = requireNonNull(left, "left is null"); - this.right = requireNonNull(right, "right is null"); - } - - @JsonProperty - public VariableReferenceExpression getLeft() - { - return left; - } - - @JsonProperty - public VariableReferenceExpression getRight() - { - return right; - } - - public EquiJoinClause flip() - { - return new EquiJoinClause(right, left); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - - if (obj == null || !this.getClass().equals(obj.getClass())) { - return false; - } - - EquiJoinClause other = (EquiJoinClause) obj; - - return Objects.equals(this.left, other.left) && - Objects.equals(this.right, other.right); - } - - @Override - public int hashCode() - { - return Objects.hash(left, right); - } - - @Override - public String toString() - { - return format("%s = %s", left, right); - } - } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java index 49ad5269dc4e8..c9105843adbee 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LateralJoinNode.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.plan; import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -42,17 +43,17 @@ public class LateralJoinNode { public enum Type { - INNER(JoinNode.Type.INNER), - LEFT(JoinNode.Type.LEFT); + INNER(JoinType.INNER), + LEFT(JoinType.LEFT); - Type(JoinNode.Type joinNodeType) + Type(JoinType joinNodeType) { this.joinNodeType = joinNodeType; } - private final JoinNode.Type joinNodeType; + private final JoinType joinNodeType; - public JoinNode.Type toJoinNodeType() + public JoinType toJoinNodeType() { return joinNodeType; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MergeJoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MergeJoinNode.java index ca3bcdd86ae0d..6ebdcdb2be711 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MergeJoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MergeJoinNode.java @@ -14,6 +14,8 @@ package com.facebook.presto.sql.planner.plan; import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.RowExpression; @@ -34,10 +36,10 @@ public class MergeJoinNode extends InternalPlanNode { - private final JoinNode.Type type; + private final JoinType type; private final PlanNode left; private final PlanNode right; - private final List criteria; + private final List criteria; private final Optional filter; private final List outputVariables; private final Optional leftHashVariable; @@ -47,10 +49,10 @@ public class MergeJoinNode public MergeJoinNode( Optional sourceLocation, @JsonProperty ("id") PlanNodeId id, - @JsonProperty("type") JoinNode.Type type, + @JsonProperty("type") JoinType type, @JsonProperty("left") PlanNode left, @JsonProperty("right") PlanNode right, - @JsonProperty("criteria") List criteria, + @JsonProperty("criteria") List criteria, @JsonProperty("outputVariables") List outputVariables, @JsonProperty("filter") Optional filter, @JsonProperty("leftHashVariable") Optional leftHashVariable, @@ -63,10 +65,10 @@ public MergeJoinNode( Optional sourceLocation, PlanNodeId id, Optional statsEquivalentPlanNode, - JoinNode.Type type, + JoinType type, PlanNode left, PlanNode right, - List criteria, + List criteria, List outputVariables, Optional filter, Optional leftHashVariable, @@ -84,7 +86,7 @@ public MergeJoinNode( } @JsonProperty - public JoinNode.Type getType() + public JoinType getType() { return type; } @@ -102,7 +104,7 @@ public PlanNode getRight() } @JsonProperty - public List getCriteria() + public List getCriteria() { return criteria; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index 94d8458a78a84..d1570edc590ce 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.OutputNode; @@ -236,7 +237,7 @@ public static Property> correlation public static class Join { - public static Property type() + public static Property type() { return property("type", JoinNode::getType); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SpatialJoinNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SpatialJoinNode.java index fd02140e2afe3..66acfa76fcffe 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SpatialJoinNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SpatialJoinNode.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.plan; import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.RowExpression; @@ -53,7 +54,7 @@ public String getJoinLabel() return joinLabel; } - public static Type fromJoinNodeType(JoinNode.Type joinNodeType) + public static Type fromJoinNodeType(JoinType joinNodeType) { switch (joinNodeType) { case INNER: diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index 44fcf13ae3ab0..a4d610351330a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -41,6 +41,7 @@ import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; @@ -502,7 +503,7 @@ public Void visitExplainAnalyze(ExplainAnalyzeNode node, Void context) public Void visitJoin(JoinNode node, Void context) { List joinExpressions = new ArrayList<>(); - for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + for (EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(JoinNodeUtils.toExpression(clause).toString()); } node.getFilter().map(formatter::apply).ifPresent(joinExpressions::add); @@ -602,7 +603,7 @@ public Void visitIndexJoin(IndexJoinNode node, Void context) public Void visitMergeJoin(MergeJoinNode node, Void context) { List joinExpressions = new ArrayList<>(); - for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + for (EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(JoinNodeUtils.toExpression(clause).toString()); } node.getFilter().map(formatter::apply).ifPresent(joinExpressions::add); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index c636076a72a4d..cb8ad04064727 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; @@ -395,7 +396,7 @@ public Void visitJoin(JoinNode node, Set boundVaria .addAll(rightInputs) .build(); - for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + for (EquiJoinClause clause : node.getCriteria()) { checkArgument(leftInputs.contains(clause.getLeft()), "Symbol from join clause (%s) not in left source (%s)", clause.getLeft(), node.getLeft().getOutputVariables()); checkArgument(rightInputs.contains(clause.getRight()), "Symbol from join clause (%s) not in right source (%s)", clause.getRight(), node.getRight().getOutputVariables()); } @@ -473,7 +474,7 @@ public Void visitMergeJoin(MergeJoinNode node, Set .addAll(rightInputs) .build(); - for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + for (EquiJoinClause clause : node.getCriteria()) { checkArgument(leftInputs.contains(clause.getLeft()), "Symbol from join clause (%s) not in left source (%s)", clause.getLeft(), node.getLeft().getOutputVariables()); checkArgument(rightInputs.contains(clause.getRight()), "Symbol from join clause (%s) not in right source (%s)", clause.getRight(), node.getRight().getOutputVariables()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingJoins.java index 80a22cd5eeec3..1c7a0ce218cac 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingJoins.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; @@ -81,7 +82,7 @@ public Void visitJoin(JoinNode node, Void context) // Validate the streaming property of the join node is satisfied when no RemoteSourceNode is involved. if (!searchFrom(node).where(RemoteSourceNode.class::isInstance).matches()) { List buildJoinVariables = node.getCriteria().stream() - .map(JoinNode.EquiJoinClause::getRight) + .map(EquiJoinClause::getRight) .collect(toImmutableList()); StreamPreferredProperties requiredBuildProperty; if (getTaskConcurrency(session) > 1) { diff --git a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index 0cd6677dd86b8..055fd998b4926 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; @@ -558,7 +559,7 @@ public Void visitEnforceSingleRow(EnforceSingleRowNode node, Void context) public Void visitJoin(JoinNode node, Void context) { List joinExpressions = new ArrayList<>(); - for (JoinNode.EquiJoinClause clause : node.getCriteria()) { + for (EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(JoinNodeUtils.toExpression(clause)); } String joinCriteria = Joiner.on(" AND ").join(joinExpressions); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java index 680229bbb3242..f51b8a2dac4f9 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java @@ -32,6 +32,9 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -226,7 +229,7 @@ public void testRepartitionedJoin() JoinNode join = join("join", ts1, ts2, - JoinNode.DistributionType.PARTITIONED, + JoinDistributionType.PARTITIONED, "orderkey", "orderkey_0"); @@ -268,7 +271,7 @@ public void testReplicatedJoin() JoinNode join = join("join", ts1, ts2, - JoinNode.DistributionType.REPLICATED, + JoinDistributionType.REPLICATED, "orderkey", "orderkey_0"); @@ -319,14 +322,14 @@ public void testMemoryCostJoinAboveJoin() "join23", ts2, ts3, - JoinNode.DistributionType.PARTITIONED, + JoinDistributionType.PARTITIONED, "key2", "key3"); JoinNode join = join( "join", ts1, join23, - JoinNode.DistributionType.PARTITIONED, + JoinDistributionType.PARTITIONED, "key1", "key2"); @@ -456,7 +459,7 @@ public void testRepartitionedJoinWithExchange() JoinNode join = join("join", remoteExchange1, localExchange, - JoinNode.DistributionType.PARTITIONED, + JoinDistributionType.PARTITIONED, "orderkey_1", "orderkey_0"); @@ -493,7 +496,7 @@ public void testReplicatedJoinWithExchange() JoinNode join = join("join", ts1, localExchange, - JoinNode.DistributionType.REPLICATED, + JoinDistributionType.REPLICATED, "orderkey", "orderkey_0"); @@ -819,19 +822,19 @@ private AggregationNode aggregation(String id, PlanNode source) * EquiJoinClause is created from symbols in form of: * symbol[0] = symbol[1] AND symbol[2] = symbol[3] AND ... */ - private JoinNode join(String planNodeId, PlanNode left, PlanNode right, JoinNode.DistributionType distributionType, String... symbols) + private JoinNode join(String planNodeId, PlanNode left, PlanNode right, JoinDistributionType distributionType, String... symbols) { checkArgument(symbols.length % 2 == 0); - ImmutableList.Builder criteria = ImmutableList.builder(); + ImmutableList.Builder criteria = ImmutableList.builder(); for (int i = 0; i < symbols.length; i += 2) { - criteria.add(new JoinNode.EquiJoinClause(new VariableReferenceExpression(Optional.empty(), symbols[i], BIGINT), new VariableReferenceExpression(Optional.empty(), symbols[i + 1], BIGINT))); + criteria.add(new EquiJoinClause(new VariableReferenceExpression(Optional.empty(), symbols[i], BIGINT), new VariableReferenceExpression(Optional.empty(), symbols[i + 1], BIGINT))); } return new JoinNode( Optional.empty(), new PlanNodeId(planNodeId), - JoinNode.Type.INNER, + JoinType.INNER, left, right, criteria.build(), diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java index 14255482733e1..0f642ae517561 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java @@ -16,10 +16,10 @@ import com.facebook.presto.Session; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -34,10 +34,10 @@ import static com.facebook.presto.cost.FilterStatsCalculator.UNKNOWN_FILTER_COEFFICIENT; import static com.facebook.presto.cost.PlanNodeStatsAssertion.assertThat; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static java.lang.Double.NaN; @@ -419,13 +419,13 @@ public void testAddJoinComplementStats() .equalTo(addedStats); } - private void assertJoinStats(JoinNode.Type joinType, PlanNodeStatsEstimate leftStats, PlanNodeStatsEstimate rightStats, PlanNodeStatsEstimate resultStats) + private void assertJoinStats(JoinType joinType, PlanNodeStatsEstimate leftStats, PlanNodeStatsEstimate rightStats, PlanNodeStatsEstimate resultStats) { assertJoinStats(joinType, LEFT_JOIN_COLUMN, LEFT_OTHER_COLUMN, RIGHT_JOIN_COLUMN, RIGHT_OTHER_COLUMN, leftStats, rightStats, resultStats); } private void assertJoinStats( - JoinNode.Type joinType, + JoinType joinType, VariableReferenceExpression leftJoinColumn, VariableReferenceExpression leftOtherColumn, VariableReferenceExpression rightJoinColumn, @@ -438,7 +438,7 @@ private void assertJoinStats( } private void assertJoinStats( - JoinNode.Type joinType, + JoinType joinType, VariableReferenceExpression leftJoinColumn, VariableReferenceExpression leftOtherColumn, VariableReferenceExpression rightJoinColumn, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java index c23f9c36d874b..31c11e1cfb8f7 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java @@ -18,6 +18,7 @@ import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; @@ -44,13 +45,13 @@ import java.util.stream.Stream; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPLICATE; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.google.common.collect.ImmutableList.toImmutableList; import static org.testng.Assert.assertEquals; @@ -226,7 +227,7 @@ private static PlanFragment createBroadcastJoinPlanFragment(String name, PlanFra return createFragment(join); } - private static PlanFragment createJoinPlanFragment(JoinNode.Type joinType, String name, PlanFragment buildFragment, PlanFragment probeFragment) + private static PlanFragment createJoinPlanFragment(JoinType joinType, String name, PlanFragment buildFragment, PlanFragment probeFragment) { RemoteSourceNode probe = new RemoteSourceNode(Optional.empty(), new PlanNodeId("probe_id"), probeFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPARTITION); RemoteSourceNode build = new RemoteSourceNode(Optional.empty(), new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPARTITION); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index 658fbc29fbaca..bef28ae62c176 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -87,10 +87,10 @@ import static com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.Integer.min; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java b/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java index f2e9bc576dbe4..92e2e380a1bfb 100644 --- a/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java @@ -33,11 +33,11 @@ import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.geospatial.SphericalGeographyType.SPHERICAL_GEOGRAPHY; import static com.facebook.presto.geospatial.type.GeometryType.GEOMETRY; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.spatialJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestExtractSpatialInnerJoin extends BaseRuleTest diff --git a/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialLeftJoin.java b/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialLeftJoin.java index 2491cabeb97c5..63dff8fb443cd 100644 --- a/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialLeftJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialLeftJoin.java @@ -33,10 +33,10 @@ import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.geospatial.SphericalGeographyType.SPHERICAL_GEOGRAPHY; import static com.facebook.presto.geospatial.type.GeometryType.GEOMETRY; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.spatialLeftJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; public class TestExtractSpatialLeftJoin extends BaseRuleTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java index c41193a009c3a..e30ce191d842a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java @@ -29,6 +29,7 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.REWRITE_EXPRESSION_WITH_CONSTANT_EXPRESSION; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; @@ -37,7 +38,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.specification; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.window; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestCanonicalize extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestDynamicFilter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestDynamicFilter.java index 4e386ab85427f..807fa01acd8ff 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestDynamicFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestDynamicFilter.java @@ -28,6 +28,9 @@ import static com.facebook.presto.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.PARTITIONED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyNot; @@ -41,9 +44,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; public class TestDynamicFilter extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java index 266fdf92a7e00..e250c4ed60fea 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEarlyOutJoins.java @@ -23,6 +23,7 @@ import static com.facebook.presto.SystemSessionProperties.IN_PREDICATES_AS_INNER_JOINS_ENABLED; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.SystemSessionProperties.PUSH_AGGREGATION_BELOW_JOIN_BYTE_REDUCTION_THRESHOLD; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -33,7 +34,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestEarlyOutJoins extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java index 6210aaf82d21e..3edb9a4759356 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java @@ -24,7 +24,9 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.Ordering; import com.facebook.presto.spi.plan.OrderingScheme; @@ -424,10 +426,10 @@ public void testUnion() @Test public void testInnerJoin() { - ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); - criteriaBuilder.add(new JoinNode.EquiJoinClause(AV, DV)); - criteriaBuilder.add(new JoinNode.EquiJoinClause(BV, EV)); - List criteria = criteriaBuilder.build(); + ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); + criteriaBuilder.add(new EquiJoinClause(AV, DV)); + criteriaBuilder.add(new EquiJoinClause(BV, EV)); + List criteria = criteriaBuilder.build(); Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); TableScanNode leftScan = tableScanNode(leftAssignments); @@ -448,7 +450,7 @@ public void testInnerJoin() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.INNER, + JoinType.INNER, left, right, criteria, @@ -490,10 +492,10 @@ public void testInnerJoinPropagatesPredicatesViaEquiConditions() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.INNER, + JoinType.INNER, left, rightScan, - ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)), + ImmutableList.of(new EquiJoinClause(AV, DV)), ImmutableList.builder() .addAll(rightScan.getOutputVariables()) .build(), @@ -522,10 +524,10 @@ public void testInnerJoinWithFalseFilter() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.INNER, + JoinType.INNER, leftScan, rightScan, - ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)), + ImmutableList.of(new EquiJoinClause(AV, DV)), ImmutableList.builder() .addAll(leftScan.getOutputVariables()) .addAll(rightScan.getOutputVariables()) @@ -544,10 +546,10 @@ public void testInnerJoinWithFalseFilter() @Test public void testLeftJoin() { - ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); - criteriaBuilder.add(new JoinNode.EquiJoinClause(AV, DV)); - criteriaBuilder.add(new JoinNode.EquiJoinClause(BV, EV)); - List criteria = criteriaBuilder.build(); + ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); + criteriaBuilder.add(new EquiJoinClause(AV, DV)); + criteriaBuilder.add(new EquiJoinClause(BV, EV)); + List criteria = criteriaBuilder.build(); Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); TableScanNode leftScan = tableScanNode(leftAssignments); @@ -567,7 +569,7 @@ public void testLeftJoin() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.LEFT, + JoinType.LEFT, left, right, criteria, @@ -596,7 +598,7 @@ public void testLeftJoin() @Test public void testLeftJoinWithFalseInner() { - List criteria = ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)); + List criteria = ImmutableList.of(new EquiJoinClause(AV, DV)); Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); TableScanNode leftScan = tableScanNode(leftAssignments); @@ -613,7 +615,7 @@ public void testLeftJoinWithFalseInner() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.LEFT, + JoinType.LEFT, left, right, criteria, @@ -639,10 +641,10 @@ public void testLeftJoinWithFalseInner() @Test public void testRightJoin() { - ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); - criteriaBuilder.add(new JoinNode.EquiJoinClause(AV, DV)); - criteriaBuilder.add(new JoinNode.EquiJoinClause(BV, EV)); - List criteria = criteriaBuilder.build(); + ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); + criteriaBuilder.add(new EquiJoinClause(AV, DV)); + criteriaBuilder.add(new EquiJoinClause(BV, EV)); + List criteria = criteriaBuilder.build(); Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); TableScanNode leftScan = tableScanNode(leftAssignments); @@ -662,7 +664,7 @@ public void testRightJoin() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.RIGHT, + JoinType.RIGHT, left, right, criteria, @@ -691,7 +693,7 @@ public void testRightJoin() @Test public void testRightJoinWithFalseInner() { - List criteria = ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)); + List criteria = ImmutableList.of(new EquiJoinClause(AV, DV)); Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); TableScanNode leftScan = tableScanNode(leftAssignments); @@ -707,7 +709,7 @@ public void testRightJoinWithFalseInner() PlanNode node = new JoinNode( Optional.empty(), newId(), - JoinNode.Type.RIGHT, + JoinType.RIGHT, left, right, criteria, diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestInequalityInferenceInJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestInequalityInferenceInJoins.java index 64d926af861c2..7694a8108cf45 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestInequalityInferenceInJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestInequalityInferenceInJoins.java @@ -21,6 +21,9 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.INFER_INEQUALITY_PREDICATES; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; @@ -28,9 +31,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; public class TestInequalityInferenceInJoins extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index 4c39bacb8bc73..5f04475dfa7db 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -77,6 +77,11 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.TestExpressionInterpreter.AVG_UDAF_CPP; @@ -120,11 +125,6 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPLICATE; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.tree.SortItem.NullOrdering.LAST; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; import static com.facebook.presto.sql.tree.SortItem.Ordering.DESCENDING; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPlanMatchingFramework.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPlanMatchingFramework.java index 57c9a5fced928..dbc1c328e7913 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPlanMatchingFramework.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPlanMatchingFramework.java @@ -23,6 +23,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.PARTITIONED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; @@ -40,7 +41,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictTableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static org.testng.Assert.fail; public class TestPlanMatchingFramework diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java index edc7b2eeb15b3..b034012260b17 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.common.function.OperatorType; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; @@ -22,7 +23,6 @@ import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.optimizations.PredicatePushDown; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -34,6 +34,10 @@ import static com.facebook.presto.SystemSessionProperties.GENERATE_DOMAIN_FILTERS; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; @@ -47,10 +51,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; import static com.facebook.presto.sql.relational.Expressions.constant; import static java.util.Collections.emptyList; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdownWithDynamicFilter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdownWithDynamicFilter.java index 87dc57c137bf6..395ba33763df3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdownWithDynamicFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdownWithDynamicFilter.java @@ -22,6 +22,7 @@ import static com.facebook.presto.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; import static com.facebook.presto.SystemSessionProperties.GENERATE_DOMAIN_FILTERS; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; @@ -32,7 +33,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestPredicatePushdownWithDynamicFilter extends TestPredicatePushdown diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPushDownDereferences.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPushDownDereferences.java index 964d06df4c3b7..2d666ca4aad41 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPushDownDereferences.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPushDownDereferences.java @@ -22,6 +22,7 @@ import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_DEREFERENCE_ENABLED; import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_SUBFIELDS_ENABLED; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.Ordering; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; @@ -38,7 +39,6 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.tree.SortItem.NullOrdering.LAST; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestSchedulingOrderVisitor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestSchedulingOrderVisitor.java index c7256d3feda04..f5caaf26a4859 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestSchedulingOrderVisitor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestSchedulingOrderVisitor.java @@ -17,13 +17,13 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.TestingColumnHandle; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.plan.IndexJoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -47,7 +47,7 @@ public void testJoinOrder() PlanBuilder planBuilder = new PlanBuilder(TEST_SESSION, new PlanNodeIdAllocator(), METADATA); TableScanNode a = planBuilder.tableScan(emptyList(), emptyMap()); TableScanNode b = planBuilder.tableScan(emptyList(), emptyMap()); - List order = scheduleOrder(planBuilder.join(JoinNode.Type.INNER, a, b)); + List order = scheduleOrder(planBuilder.join(JoinType.INNER, a, b)); assertEquals(order, ImmutableList.of(b.getId(), a.getId())); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestUseDefaultsforCorrelatedAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestUseDefaultsforCorrelatedAggregations.java index 846b969f540d6..bd375f799ec72 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestUseDefaultsforCorrelatedAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestUseDefaultsforCorrelatedAggregations.java @@ -20,6 +20,8 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; @@ -30,8 +32,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; public class TestUseDefaultsforCorrelatedAggregations extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/EquiJoinClauseProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/EquiJoinClauseProvider.java index 1bbb0e2ce8a8f..061afbe062145 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/EquiJoinClauseProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/EquiJoinClauseProvider.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.sql.planner.assertions; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.JoinNode; import java.util.Optional; @@ -22,7 +22,7 @@ import static java.util.Objects.requireNonNull; class EquiJoinClauseProvider - implements ExpectedValueProvider + implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; @@ -33,9 +33,9 @@ class EquiJoinClauseProvider this.right = requireNonNull(right, "right is null"); } - public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) + public EquiJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.EquiJoinClause( + return new EquiJoinClause( new VariableReferenceExpression(Optional.empty(), left.toSymbol(aliases).getName(), BIGINT), new VariableReferenceExpression(Optional.empty(), right.toSymbol(aliases).getName(), BIGINT)); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/JoinMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/JoinMatcher.java index 029edf2734188..73b49fdc017f1 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/JoinMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/JoinMatcher.java @@ -16,10 +16,12 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.DistributionType; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; @@ -36,17 +38,17 @@ final class JoinMatcher implements Matcher { - private final JoinNode.Type joinType; - private final List> equiCriteria; + private final JoinType joinType; + private final List> equiCriteria; private final Optional filter; - private final Optional distributionType; + private final Optional distributionType; private final Optional dynamicFilter; JoinMatcher( - JoinNode.Type joinType, - List> equiCriteria, + JoinType joinType, + List> equiCriteria, Optional filter, - Optional distributionType, + Optional distributionType, Optional dynamicFilter) { this.joinType = requireNonNull(joinType, "joinType is null"); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/MergeJoinMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/MergeJoinMatcher.java index 0d4beb89f1e4e..57c363ebb0c70 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/MergeJoinMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/MergeJoinMatcher.java @@ -16,9 +16,10 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.MergeJoinNode; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; @@ -36,13 +37,13 @@ public class MergeJoinMatcher implements Matcher { - private final JoinNode.Type joinType; - private final List> equiCriteria; + private final JoinType joinType; + private final List> equiCriteria; private final Optional filter; MergeJoinMatcher( - JoinNode.Type joinType, - List> equiCriteria, + JoinType joinType, + List> equiCriteria, Optional filter) { this.joinType = requireNonNull(joinType, "joinType is null"); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java index 34f3d6124615a..20fc5b8643c87 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java @@ -23,9 +23,12 @@ import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.OutputNode; @@ -397,17 +400,17 @@ public static PlanMatchPattern join(PlanMatchPattern left, PlanMatchPattern righ return node(JoinNode.class, left, right); } - public static PlanMatchPattern join(JoinNode.Type joinType, List> expectedEquiCriteria, PlanMatchPattern left, PlanMatchPattern right) + public static PlanMatchPattern join(JoinType joinType, List> expectedEquiCriteria, PlanMatchPattern left, PlanMatchPattern right) { return join(joinType, expectedEquiCriteria, Optional.empty(), left, right); } - public static PlanMatchPattern join(JoinNode.Type joinType, List> expectedEquiCriteria, Optional expectedFilter, PlanMatchPattern left, PlanMatchPattern right) + public static PlanMatchPattern join(JoinType joinType, List> expectedEquiCriteria, Optional expectedFilter, PlanMatchPattern left, PlanMatchPattern right) { return join(joinType, expectedEquiCriteria, expectedFilter, Optional.empty(), left, right); } - public static PlanMatchPattern join(JoinNode.Type joinType, List> expectedEquiCriteria, Optional expectedFilter, Optional expectedDistributionType, PlanMatchPattern left, PlanMatchPattern right) + public static PlanMatchPattern join(JoinType joinType, List> expectedEquiCriteria, Optional expectedFilter, Optional expectedDistributionType, PlanMatchPattern left, PlanMatchPattern right) { return node(JoinNode.class, left, right).with( new JoinMatcher( @@ -419,8 +422,8 @@ public static PlanMatchPattern join(JoinNode.Type joinType, List> expectedEquiCriteria, + JoinType joinType, + List> expectedEquiCriteria, Map expectedDynamicFilter, Optional expectedStaticFilter, PlanMatchPattern leftSource, @@ -476,7 +479,7 @@ public static PlanMatchPattern spatialLeftJoin(String expectedFilter, PlanMatchP new SpatialJoinMatcher(SpatialJoinNode.Type.LEFT, rewriteIdentifiersToSymbolReferences(new SqlParser().createExpression(expectedFilter, new ParsingOptions())), Optional.empty())); } - public static PlanMatchPattern mergeJoin(JoinNode.Type joinType, List> expectedEquiCriteria, Optional filter, PlanMatchPattern left, PlanMatchPattern right) + public static PlanMatchPattern mergeJoin(JoinType joinType, List> expectedEquiCriteria, Optional filter, PlanMatchPattern left, PlanMatchPattern right) { return node(MergeJoinNode.class, left, right).with( new MergeJoinMatcher( @@ -532,7 +535,7 @@ public static PlanMatchPattern except(PlanMatchPattern... sources) return node(ExceptNode.class, sources); } - public static ExpectedValueProvider equiJoinClause(String left, String right) + public static ExpectedValueProvider equiJoinClause(String left, String right) { return new EquiJoinClauseProvider(new SymbolAlias(left), new SymbolAlias(right)); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddNotNullFiltersToJoinNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddNotNullFiltersToJoinNode.java index 9ab1356677199..bc54b3ff5230b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddNotNullFiltersToJoinNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddNotNullFiltersToJoinNode.java @@ -42,6 +42,9 @@ import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_NULLS_IN_JOINS; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinNotNullInferenceStrategy.INFER_FROM_STANDARD_OPERATORS; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinNotNullInferenceStrategy.USE_FUNCTION_METADATA; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -49,9 +52,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.relational.Expressions.variable; import static com.facebook.presto.testing.assertions.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayContainsToInnerJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayContainsToInnerJoin.java index c7b05dd049037..282a031c8f04a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayContainsToInnerJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayContainsToInnerJoin.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -47,13 +47,13 @@ public void testTriggerForBigInt() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("contains(left_array_k1, right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT))), p.values(p.variable("right_k1")))); }) .matches( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("field", "right_k1")), unnest( ImmutableMap.of("array_distinct", ImmutableList.of("field")), @@ -74,13 +74,13 @@ public void testTriggerForBigIntArrayRightSide() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(right_array_k1, left_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1")), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }) .matches( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("left_k1", "field")), values("left_k1"), unnest( @@ -103,7 +103,7 @@ public void testMultipleArrayContainsConditions() p.variable("right_array_k2", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(left_array_k1, right_k1) and contains(right_array_k2, left_k2)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_array_k2", new ArrayType(BIGINT))))); }) @@ -111,7 +111,7 @@ public void testMultipleArrayContainsConditions() filter( "contains(right_array_k2, left_k2)", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("field", "right_k1")), unnest( ImmutableMap.of("array_distinct", ImmutableList.of("field")), @@ -134,7 +134,7 @@ public void testMultipleInvalidArrayContainsConditions() p.variable("right_array_k2", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(left_array_k1, right_k1) or contains(right_array_k2, left_k2)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_array_k2", new ArrayType(BIGINT))))); }).doesNotFire(); @@ -151,7 +151,7 @@ public void testNotTriggerForDouble() p.variable("right_k1", DOUBLE); return p.filter( p.rowExpression("contains(left_array_k1, right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(DOUBLE))), p.values(p.variable("right_k1", DOUBLE)))); }).doesNotFire(); @@ -168,7 +168,7 @@ public void testArrayContainsWithCast() p.variable("right_k1", VARCHAR); return p.filter( p.rowExpression("contains(left_array_k1, CAST(right_k1 AS BIGINT))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", new ArrayType(BIGINT))), p.values(p.variable("right_k1", VARCHAR)))); }).doesNotFire(); @@ -188,14 +188,14 @@ public void testArrayContainsWithCastBothRules() p.variable("right_k1", VARCHAR); return p.filter( p.rowExpression("contains(left_array_k1, CAST(right_k1 AS BIGINT))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT))), p.values(p.variable("right_k1", VARCHAR)))); }) .matches( project( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("field", "cast_r")), unnest( ImmutableMap.of("array_distinct", ImmutableList.of("field")), @@ -221,14 +221,14 @@ public void testArrayContainsWithCastBothRules2() p.variable("right_k1", VARCHAR); return p.filter( p.rowExpression("contains(CAST(left_array_k1 AS ARRAY), right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT))), p.values(p.variable("right_k1", VARCHAR)))); }) .matches( project( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("field", "right_k1")), unnest( ImmutableMap.of("array_distinct", ImmutableList.of("field")), @@ -254,14 +254,14 @@ public void testArrayContainsWithCastBothRulesArrayRightSide() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(right_array_k1, CAST(left_k1 AS BIGINT))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }) .matches( project( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("cast_l", "field")), project( ImmutableMap.of("cast_l", expression("CAST(left_k1 AS bigint)")), @@ -288,14 +288,14 @@ public void testArrayContainsWithCoalesceBothRules() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("contains(left_array_k1, coalesce(CAST(right_k1 AS BIGINT), right_k2))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT))), p.values(p.variable("right_k1", VARCHAR), p.variable("right_k2", BIGINT)))); }) .matches( project( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("field", "expr")), unnest( ImmutableMap.of("array_distinct", ImmutableList.of("field")), @@ -322,14 +322,14 @@ public void testArrayContainsWithCoalesceBothRulesArrayOnRight() p.variable("left_k2", BIGINT); return p.filter( p.rowExpression("contains(right_array_k1, coalesce(CAST(left_k1 AS BIGINT), left_k2))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR), p.variable("left_k2", BIGINT)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }) .matches( project( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("expr", "field")), project( ImmutableMap.of("expr", expression("COALESCE(CAST(left_k1 AS bigint), left_k2)")), @@ -354,7 +354,7 @@ public void testConditionWithAnd() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("contains(left_array_k1, right_k1) and left_k2+right_k2 > 10"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) @@ -362,7 +362,7 @@ public void testConditionWithAnd() filter( "left_k2+right_k2 > 10", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("field", "right_k1")), unnest( ImmutableMap.of("array_distinct", ImmutableList.of("field")), @@ -385,7 +385,7 @@ public void testConditionWithAndArrayOnRight() p.variable("left_k2", BIGINT); return p.filter( p.rowExpression("contains(right_array_k1, left_k1) and right_k2+left_k2 > 10"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_array_k1", new ArrayType(BIGINT)), p.variable("right_k2")))); }) @@ -393,7 +393,7 @@ public void testConditionWithAndArrayOnRight() filter( "right_k2+left_k2 > 10", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("left_k1", "field")), values("left_k1", "left_k2"), unnest( @@ -416,7 +416,7 @@ public void testNonMatchingCondition() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("contains(left_array_k1, right_k1) or left_k2+right_k2 > 10"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }).doesNotFire(); @@ -435,7 +435,7 @@ public void testNonMatchingCondition2() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("contains(left_array_k1, left_k2)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }).doesNotFire(); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java index f225cf8a2a109..4ed9fc57ddb8e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java @@ -16,6 +16,7 @@ import com.facebook.presto.common.type.ArrayType; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; @@ -47,7 +48,7 @@ public void testTriggerForBigInt() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("not contains(left_array_k1, right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.enforceSingleRow(p.values(p.variable("left_array_k1", new ArrayType(BIGINT)))), p.values(p.variable("right_k1")))); }) @@ -71,7 +72,7 @@ public void testTriggerForBigIntArrayRightSide() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("not contains(right_array_k1, left_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1")), p.enforceSingleRow(p.values(p.variable("right_array_k1", new ArrayType(BIGINT)))))); }) @@ -95,7 +96,7 @@ public void testMultipleConditions() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("not contains(left_array_k1, right_k1) and right_k2>1"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.enforceSingleRow(p.values(p.variable("left_array_k1", new ArrayType(BIGINT)))), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) @@ -124,7 +125,7 @@ public void testComputedArrayExpression() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("not contains(cast(json_parse(json_string) as array), right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.project(p.enforceSingleRow(p.values(p.variable("x", VARCHAR))), Assignments.builder() .put(p.variable("json_string", VARCHAR), p.rowExpression("x")) @@ -156,7 +157,7 @@ public void testMultipleInvalidArrayNotContainsConditions() p.variable("right_array_k2", new ArrayType(BIGINT)); return p.filter( p.rowExpression("not contains(left_array_k1, right_k1) or not contains(right_array_k2, left_k2)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.enforceSingleRow(p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2"))), p.values(p.variable("right_k1"), p.variable("right_array_k2", new ArrayType(BIGINT))))); }).doesNotFire(); @@ -173,7 +174,7 @@ public void testNotTriggerForDouble() p.variable("right_k1", DOUBLE); return p.filter( p.rowExpression("not contains(left_array_k1, right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.enforceSingleRow(p.values(p.variable("left_array_k1", new ArrayType(DOUBLE)))), p.values(p.variable("right_k1", DOUBLE)))); }).doesNotFire(); @@ -190,7 +191,7 @@ public void testArrayContainsWithCast() p.variable("right_k1", VARCHAR); return p.filter( p.rowExpression("not contains(left_array_k1, CAST(right_k1 AS BIGINT))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.enforceSingleRow(p.values(p.variable("left_k1", new ArrayType(BIGINT)))), p.values(p.variable("right_k1", VARCHAR)))); }).doesNotFire(); @@ -207,7 +208,7 @@ public void testNotTriggerForMultiRow() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("not contains(left_array_k1, right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(ImmutableList.of(p.variable("left_array_k1", new ArrayType(BIGINT))), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), p.values(p.variable("right_k1")))); @@ -228,7 +229,7 @@ public void testNotTriggerForMultipleColumnsOnArraySide() p.variable("right_array_k2", new ArrayType(BIGINT)); return p.filter( p.rowExpression("not contains(left_array_k1, right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_array_k2", new ArrayType(BIGINT))))); }) @@ -247,7 +248,7 @@ public void testNotTriggerForNonDeterministicArrayExpression() p.variable("right_k1", VARCHAR); return p.filter( p.rowExpression("not contains(transform(left_array_k1, x->lower(x)), right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.enforceSingleRow(p.values(p.variable("left_array_k1", new ArrayType(VARCHAR)))), p.values(p.variable("right_k1", VARCHAR)))); }) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithOrFilterToInnerJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithOrFilterToInnerJoin.java index 3aa2aa3321a37..bd4bfc4566180 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithOrFilterToInnerJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithOrFilterToInnerJoin.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -46,7 +46,7 @@ public void testTriggerForBigInt() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("left_k1 = right_k1 or left_k2 = right_k2"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) @@ -55,7 +55,7 @@ public void testTriggerForBigInt() filter( "case left_idx when 1 then left_k1 = right_k1 when 2 then not(coalesce(left_k1 = right_k1, false)) and left_k2 = right_k2 else null end", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("expr", "expr_2"), equiJoinClause("left_idx", "right_idx")), project( ImmutableMap.of("expr", expression("case left_idx when 1 then left_k1 when 2 then left_k2 else null end")), @@ -87,7 +87,7 @@ public void testMultipleOrConditions() p.variable("right_k3", BIGINT); return p.filter( p.rowExpression("left_k1 = right_k1 or left_k2 = right_k2 or left_k3 = right_k3"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2"), p.variable("left_k3")), p.values(p.variable("right_k1"), p.variable("right_k2"), p.variable("right_k3")))); }) @@ -97,7 +97,7 @@ public void testMultipleOrConditions() "case left_idx when 1 then left_k1 = right_k1 when 2 then not(coalesce(left_k1 = right_k1, false)) and left_k2 = right_k2 " + "when 3 then not(coalesce(left_k1 = right_k1, false)) and not(coalesce(left_k2 = right_k2, false)) and left_k3 = right_k3 else null end", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("expr", "expr_2"), equiJoinClause("left_idx", "right_idx")), project( ImmutableMap.of("expr", expression("case left_idx when 1 then left_k1 when 2 then left_k2 when 3 then left_k3 else null end")), @@ -127,7 +127,7 @@ public void testNotTriggerForDouble() p.variable("right_k2", DOUBLE); return p.filter( p.rowExpression("left_k1 = right_k1 or left_k2 = right_k2"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", DOUBLE), p.variable("left_k2", DOUBLE)), p.values(p.variable("right_k1", DOUBLE), p.variable("right_k2", DOUBLE)))); }).doesNotFire(); @@ -145,7 +145,7 @@ public void testNotTriggerForCastToDouble() p.variable("right_k2", VARCHAR); return p.filter( p.rowExpression("left_k1 = right_k1 or CAST(left_k2 AS DOUBLE) = CAST(right_k2 AS DOUBLE)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR), p.variable("left_k2", VARCHAR)), p.values(p.variable("right_k1", VARCHAR), p.variable("right_k2", VARCHAR)))); }).doesNotFire(); @@ -163,7 +163,7 @@ public void testOrWithCast() p.variable("right_k2", VARCHAR); return p.filter( p.rowExpression("left_k1 = right_k1 or left_k2 = CAST(right_k2 AS BIGINT)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2", VARCHAR)))); }).doesNotFire(); @@ -184,7 +184,7 @@ public void testOrWithCastBothRules() p.variable("right_k2", VARCHAR); return p.filter( p.rowExpression("left_k1 = right_k1 or left_k2 = CAST(right_k2 AS BIGINT)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2", VARCHAR)))); }) @@ -194,7 +194,7 @@ public void testOrWithCastBothRules() filter( "case left_idx when 1 then left_k1 = right_k1 when 2 then not(coalesce(left_k1 = right_k1, false)) and left_k2 = cast_0 else null end", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("expr", "expr_2"), equiJoinClause("left_idx", "right_idx")), project( ImmutableMap.of("expr", expression("case left_idx when 1 then left_k1 when 2 then left_k2 else null end")), @@ -226,7 +226,7 @@ public void testConditionWithAnd() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("(left_k1 = right_k1 or left_k2 = right_k2) and left_k1+right_k2 > 10"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) @@ -237,7 +237,7 @@ public void testConditionWithAnd() filter( "case left_idx when 1 then left_k1 = right_k1 when 2 then not(coalesce(left_k1 = right_k1, false)) and left_k2 = right_k2 else null end", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("expr", "expr_2"), equiJoinClause("left_idx", "right_idx")), project( ImmutableMap.of("expr", expression("case left_idx when 1 then left_k1 when 2 then left_k2 else null end")), @@ -267,7 +267,7 @@ public void testNonMatchingCondition() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("(left_k1 = right_k1 or left_k2 = right_k2) or left_k1+right_k2 > 10"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) @@ -286,7 +286,7 @@ public void testNonMatchingCondition2() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("left_k1 = right_k1 or left_k2 > right_k2"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java index 5bb856cadcc1e..986214cc98558 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java @@ -19,7 +19,9 @@ import com.facebook.presto.cost.TaskCountEstimator; import com.facebook.presto.cost.VariableStatsEstimate; import com.facebook.presto.spi.TestingColumnHandle; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -30,9 +32,6 @@ import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.DistributionType; -import com.facebook.presto.sql.planner.plan.JoinNode.Type; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; @@ -49,6 +48,12 @@ import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.enforceSingleRow; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; @@ -58,12 +63,6 @@ import static com.facebook.presto.sql.planner.iterative.rule.DetermineJoinDistributionType.getSourceTablesSizeInBytes; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.getFirstKnownOutputSizeInBytes; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static java.lang.Double.NaN; import static org.testng.Assert.assertEquals; @@ -91,20 +90,20 @@ public void tearDown() @Test public void testDetermineDistributionType() { - testDetermineDistributionType(JoinDistributionType.PARTITIONED, INNER, DistributionType.PARTITIONED); - testDetermineDistributionType(JoinDistributionType.BROADCAST, INNER, DistributionType.REPLICATED); - testDetermineDistributionType(JoinDistributionType.AUTOMATIC, INNER, DistributionType.PARTITIONED); + testDetermineDistributionType(JoinDistributionType.PARTITIONED, INNER, com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED); + testDetermineDistributionType(JoinDistributionType.BROADCAST, INNER, com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED); + testDetermineDistributionType(JoinDistributionType.AUTOMATIC, INNER, com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED); } @Test public void testDetermineDistributionTypeForLeftOuter() { - testDetermineDistributionType(JoinDistributionType.PARTITIONED, LEFT, DistributionType.PARTITIONED); - testDetermineDistributionType(JoinDistributionType.BROADCAST, LEFT, DistributionType.REPLICATED); - testDetermineDistributionType(JoinDistributionType.AUTOMATIC, LEFT, DistributionType.PARTITIONED); + testDetermineDistributionType(JoinDistributionType.PARTITIONED, LEFT, com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED); + testDetermineDistributionType(JoinDistributionType.BROADCAST, LEFT, com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED); + testDetermineDistributionType(JoinDistributionType.AUTOMATIC, LEFT, com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED); } - private void testDetermineDistributionType(JoinDistributionType sessionDistributedJoin, Type joinType, DistributionType expectedDistribution) + private void testDetermineDistributionType(JoinDistributionType sessionDistributedJoin, JoinType joinType, com.facebook.presto.spi.plan.JoinDistributionType expectedDistribution) { assertDetermineJoinDistributionType() .on(p -> @@ -116,7 +115,7 @@ private void testDetermineDistributionType(JoinDistributionType sessionDistribut p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, sessionDistributedJoin.name()) @@ -140,7 +139,7 @@ public void testRepartitionRightOuter() testRepartitionRightOuter(JoinDistributionType.AUTOMATIC, RIGHT); } - private void testRepartitionRightOuter(JoinDistributionType sessionDistributedJoin, Type joinType) + private void testRepartitionRightOuter(JoinDistributionType sessionDistributedJoin, JoinType joinType) { assertDetermineJoinDistributionType() .on(p -> @@ -152,7 +151,7 @@ private void testRepartitionRightOuter(JoinDistributionType sessionDistributedJo p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, sessionDistributedJoin.name()) @@ -160,7 +159,7 @@ private void testRepartitionRightOuter(JoinDistributionType sessionDistributedJo joinType, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), - Optional.of(DistributionType.PARTITIONED), + Optional.of(com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0)))); } @@ -179,7 +178,7 @@ public void testReplicateScalar() p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L)))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.PARTITIONED.name()) @@ -187,7 +186,7 @@ public void testReplicateScalar() INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), - Optional.of(DistributionType.REPLICATED), + Optional.of(com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED), values(ImmutableMap.of("A1", 0)), enforceSingleRow(values(ImmutableMap.of("B1", 0))))); } @@ -199,7 +198,7 @@ public void testReplicateNoEquiCriteria() testReplicateNoEquiCriteria(LEFT); } - private void testReplicateNoEquiCriteria(Type joinType) + private void testReplicateNoEquiCriteria(JoinType joinType) { assertDetermineJoinDistributionType() .on(p -> @@ -219,7 +218,7 @@ private void testReplicateNoEquiCriteria(Type joinType) joinType, ImmutableList.of(), Optional.of("A1 * B1 > 100"), - Optional.of(DistributionType.REPLICATED), + Optional.of(com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0)))); } @@ -237,12 +236,12 @@ public void testRetainDistributionType() p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), Optional.empty(), Optional.empty(), - Optional.of(DistributionType.REPLICATED), + Optional.of(com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED), ImmutableMap.of())) .doesNotFire(); } @@ -267,7 +266,7 @@ public void testFlipAndReplicateWhenOneTableMuchSmaller() INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -301,7 +300,7 @@ public void testFlipAndReplicateWhenOneTableMuchSmallerAndJoinCardinalityUnknown INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -333,7 +332,7 @@ public void testPartitionWhenRequiredBySession() INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.PARTITIONED.name()) @@ -366,7 +365,7 @@ public void testPartitionWhenBothTablesEqual() INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -398,7 +397,7 @@ public void testReplicatesWhenRequiredBySession() INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.BROADCAST.name()) @@ -431,7 +430,7 @@ public void testPartitionFullOuterJoin() FULL, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -463,7 +462,7 @@ public void testPartitionRightOuterJoin() RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -495,7 +494,7 @@ public void testReplicateLeftOuterJoin() LEFT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -527,7 +526,7 @@ public void testFlipAndReplicateRightOuterJoin() RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -561,7 +560,7 @@ public void testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown() RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -599,7 +598,7 @@ public void testReplicatesWhenNotRestricted() INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -630,7 +629,7 @@ public void testReplicatesWhenNotRestricted() INNER, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .matches(join( @@ -752,7 +751,7 @@ public void testReplicatesWhenSourceIsSmall() INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.filter(new PlanNodeId("filterB"), TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), bRows, b1)), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -778,7 +777,7 @@ public void testReplicatesWhenSourceIsSmall() INNER, p.filter(new PlanNodeId("filterB"), TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), bRows, b1)), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -804,7 +803,7 @@ public void testReplicatesWhenSourceIsSmall() LEFT, p.filter(new PlanNodeId("filterB"), TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), bRows, b1)), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -853,7 +852,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() INNER, p.values(new PlanNodeId("valuesB"), bRows, b1), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -878,7 +877,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() INNER, p.values(new PlanNodeId("valuesB"), bRows, b1), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -904,7 +903,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -929,7 +928,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -954,7 +953,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -980,7 +979,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() LEFT, p.values(new PlanNodeId("valuesB"), bRows, b1), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -1005,7 +1004,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() LEFT, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -1030,7 +1029,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() LEFT, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -1055,7 +1054,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() RIGHT, p.values(new PlanNodeId("valuesB"), bRows, b1), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -1080,7 +1079,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() RIGHT, p.values(new PlanNodeId("valuesB"), bRows, b1), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -1105,7 +1104,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() RIGHT, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -1130,7 +1129,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() FULL, p.values(new PlanNodeId("valuesB"), bRows, b1), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -1155,7 +1154,7 @@ public void testReplicatesWhenOneSourceIsSmallAndTheOtherUnknown() FULL, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -1208,7 +1207,7 @@ public void testFlipWhenSizeDifferenceLarge() new PlanNodeId("filterB"), TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), bRows, b1)), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty()); }) @@ -1237,7 +1236,7 @@ public void testFlipWhenSizeDifferenceLarge() TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), bRows, b1)), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) @@ -1301,7 +1300,7 @@ public void testFlipWhenSizeDifferenceLarge() TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), aRows, b1)), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty()); }) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java index 3042b2cd5b930..f0194625ba4bb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; @@ -24,7 +26,6 @@ import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.optimizations.joins.JoinGraph; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -36,13 +37,13 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.common.function.OperatorType.NEGATION; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.iterative.rule.EliminateCrossJoins.getJoinOrder; import static com.facebook.presto.sql.planner.iterative.rule.EliminateCrossJoins.isOriginalOrder; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.relational.Expressions.callOperator; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Iterables.getOnlyElement; @@ -91,7 +92,7 @@ public void testDoNotReorderOuterJoin() { tester().assertThat(new EliminateCrossJoins()) .setSystemProperty(JOIN_REORDERING_STRATEGY, "ELIMINATE_CROSS_JOINS") - .on(crossJoinAndJoin(JoinNode.Type.LEFT)) + .on(crossJoinAndJoin(JoinType.LEFT)) .doesNotFire(); } @@ -226,7 +227,7 @@ public void testGiveUpOnNonIdentityProjections() assertEquals(JoinGraph.buildFrom(plan).size(), 2); } - private Function crossJoinAndJoin(JoinNode.Type secondJoinType) + private Function crossJoinAndJoin(JoinType secondJoinType) { return p -> { VariableReferenceExpression axVariable = p.variable("axVariable"); @@ -261,16 +262,16 @@ private VariableReferenceExpression variable(String name) private JoinNode joinNode(PlanNode left, PlanNode right, VariableReferenceExpression... variables) { checkArgument(variables.length % 2 == 0); - ImmutableList.Builder criteria = ImmutableList.builder(); + ImmutableList.Builder criteria = ImmutableList.builder(); for (int i = 0; i < variables.length; i += 2) { - criteria.add(new JoinNode.EquiJoinClause(variables[i], variables[i + 1])); + criteria.add(new EquiJoinClause(variables[i], variables[i + 1])); } return new JoinNode( Optional.empty(), idAllocator.getNextId(), - JoinNode.Type.INNER, + JoinType.INNER, left, right, criteria.build(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java index e7f96e3151763..2e107375df496 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java @@ -16,6 +16,7 @@ import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -27,7 +28,6 @@ import com.facebook.presto.sql.planner.iterative.rule.ReorderJoins.MultiJoinNode; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.testing.LocalQueryRunner; @@ -45,12 +45,12 @@ import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; import static com.facebook.presto.sql.planner.iterative.rule.ReorderJoins.MultiJoinNode.toMultiJoinNode; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinNullFilterToSemiJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinNullFilterToSemiJoin.java index f74d018e48abf..bedc64ed1479b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinNullFilterToSemiJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinNullFilterToSemiJoin.java @@ -13,8 +13,9 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import org.testng.annotations.Test; import static com.facebook.presto.common.type.BigintType.BIGINT; @@ -37,10 +38,10 @@ public void testTrigger() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is null"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }) .matches( project( @@ -65,11 +66,11 @@ public void testNotTriggerWithFilter() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is null"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), p.rowExpression("left_k2 > 10"), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }).doesNotFire(); } @@ -84,10 +85,10 @@ public void testNotTriggerNotNull() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is not null"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }).doesNotFire(); } @@ -103,10 +104,10 @@ public void testNotTriggerOtherOutputUsed() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("right_k1 is null"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }).doesNotFire(); } @@ -121,10 +122,10 @@ public void testNotTriggerOutputUsedInFilter() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is null or right_k1 > 2"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }).doesNotFire(); } @@ -139,10 +140,10 @@ public void testNotTriggerOutputUsedInFilter2() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is null and right_k1 > 2"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }).doesNotFire(); } @@ -157,10 +158,10 @@ public void testNotTriggerOtherOutputUsedInFilter() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is null or left_k2 > 2"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }).doesNotFire(); } @@ -175,10 +176,10 @@ public void testTriggerForFilterWithAnd() p.variable("right_k1", BIGINT); return p.filter( p.rowExpression("right_k1 is null and left_k2 > 2"), - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1")), - new JoinNode.EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); + new EquiJoinClause(p.variable("left_k1"), p.variable("right_k1")))); }) .matches( filter( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java index 3e51a08d13443..657093aec5be2 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -46,13 +46,13 @@ public void testTriggerForBigIntArrayRightSide() p.variable("left_k1", BIGINT); p.variable("right_array_k1", new ArrayType(BIGINT)); return - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1")), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))), p.rowExpression("contains(right_array_k1, left_k1)")); }).matches( join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("left_k1", "unnest")), values("left_k1"), unnest( @@ -72,7 +72,7 @@ public void testNotTriggerForArrayOnLeftSide() p.variable("left_array_k1", new ArrayType(BIGINT)); p.variable("right_k1", BIGINT); return - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_array_k1", new ArrayType(BIGINT))), p.values(p.variable("right_k1")), p.rowExpression("contains(left_array_k1, right_k1)")); @@ -91,13 +91,13 @@ public void testMultipleArrayContainsConditions() p.variable("right_k1", BIGINT); p.variable("right_array_k2", new ArrayType(BIGINT)); return - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_array_k2", new ArrayType(BIGINT))), p.rowExpression("contains(left_array_k1, right_k1) and contains(right_array_k2, left_k2)")); }).matches( join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("left_k2", "unnest")), Optional.of("contains(left_array_k1, right_k1)"), values("left_array_k1", "left_k2"), @@ -120,7 +120,7 @@ public void testMultipleInvalidArrayContainsConditions() p.variable("right_k1", BIGINT); p.variable("right_array_k2", new ArrayType(BIGINT)); return - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_array_k1", new ArrayType(BIGINT)), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_array_k2", new ArrayType(BIGINT))), p.rowExpression("contains(left_array_k1, right_k1) or contains(right_array_k2, left_k2)")); @@ -136,13 +136,13 @@ public void testArrayContainsWithCast() { p.variable("right_array_k1", new ArrayType(BIGINT)); p.variable("left_k1", VARCHAR); - return p.join(JoinNode.Type.LEFT, + return p.join(JoinType.LEFT, p.values(p.variable("left_k1", VARCHAR)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))), p.rowExpression("contains(right_array_k1, CAST(left_k1 AS BIGINT))")); }).matches( join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("cast_left", "unnest")), project( ImmutableMap.of("cast_left", expression("CAST(left_k1 AS BIGINT)")), @@ -163,13 +163,13 @@ public void testArrayContainsWithCast2() { p.variable("right_array_k1", new ArrayType(BIGINT)); p.variable("left_k1", VARCHAR); - return p.join(JoinNode.Type.LEFT, + return p.join(JoinType.LEFT, p.values(p.variable("left_k1", VARCHAR)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))), p.rowExpression("contains(CAST(right_array_k1 AS ARRAY), left_k1)")); }).matches( join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("left_k1", "unnest")), values("left_k1"), unnest( @@ -192,13 +192,13 @@ public void testArrayContainsWithCoalesce() p.variable("left_k1", VARCHAR); p.variable("left_k2", BIGINT); return - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1", VARCHAR), p.variable("left_k2", BIGINT)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))), p.rowExpression("contains(right_array_k1, coalesce(CAST(left_k1 AS BIGINT), left_k2))")); }).matches( join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("expr", "unnest")), project( ImmutableMap.of("expr", expression("COALESCE(CAST(left_k1 AS bigint), left_k2)")), @@ -222,13 +222,13 @@ public void testConditionWithAnd() p.variable("left_k1", BIGINT); p.variable("left_k2", BIGINT); return - p.join(JoinNode.Type.LEFT, + p.join(JoinType.LEFT, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_array_k1", new ArrayType(BIGINT)), p.variable("right_k2")), p.rowExpression("contains(right_array_k1, left_k1) and right_k2+left_k2 > 10")); }).matches( join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("left_k1", "unnest")), Optional.of("right_k2+left_k2 > 10"), values("left_k1", "left_k2"), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java index 323fb1133f924..5ad4cc8b14bde 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java @@ -23,7 +23,9 @@ import com.facebook.presto.spi.constraints.TableConstraint; import com.facebook.presto.spi.constraints.UniqueConstraint; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.LogicalProperties; import com.facebook.presto.spi.plan.PlanNode; @@ -758,11 +760,11 @@ public void testJoinNodeLogicalProperties() p.variable(shipPriorityVariable); p.variable(mktSegmentVariable); - return p.join(JoinNode.Type.INNER, + return p.join(JoinType.INNER, p.limit(5, ordersTableScan), p.filter(p.rowExpression("c_mktsegment = 'BUILDING'"), customerTableScan), p.rowExpression("o_shippriority = 10"), - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -795,11 +797,11 @@ public void testJoinNodeLogicalProperties() p.variable(shipPriorityVariable); p.variable(mktSegmentVariable); - return p.join(JoinNode.Type.INNER, + return p.join(JoinType.INNER, p.limit(1, ordersTableScan), p.filter(p.rowExpression("c_mktsegment = 'BUILDING'"), customerTableScan), p.rowExpression("o_shippriority = 10"), - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -833,11 +835,11 @@ public void testJoinNodeLogicalProperties() p.variable(shipPriorityVariable); p.variable(mktSegmentVariable); - return p.join(JoinNode.Type.INNER, + return p.join(JoinType.INNER, p.filter(p.rowExpression("c_mktsegment = 'BUILDING'"), customerTableScan), p.limit(6, ordersTableScan), p.rowExpression("o_shippriority = 10"), - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -870,11 +872,11 @@ public void testJoinNodeLogicalProperties() p.variable(shipPriorityVariable); p.variable(mktSegmentVariable); - return p.join(JoinNode.Type.INNER, + return p.join(JoinType.INNER, p.filter(p.rowExpression("c_mktsegment = 'BUILDING'"), customerTableScan), p.limit(1, ordersTableScan), p.rowExpression("o_shippriority = 10"), - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -903,8 +905,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.LEFT, p.limit(7, ordersTableScan), customerTableScan, - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + return p.join(JoinType.LEFT, p.limit(7, ordersTableScan), customerTableScan, + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -933,8 +935,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.LEFT, ordersTableScan, p.limit(8, customerTableScan), - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + return p.join(JoinType.LEFT, ordersTableScan, p.limit(8, customerTableScan), + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -963,8 +965,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.RIGHT, customerTableScan, p.limit(9, ordersTableScan), - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + return p.join(JoinType.RIGHT, customerTableScan, p.limit(9, ordersTableScan), + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -993,8 +995,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.RIGHT, p.limit(10, customerTableScan), ordersTableScan, - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + return p.join(JoinType.RIGHT, p.limit(10, customerTableScan), ordersTableScan, + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -1022,8 +1024,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.LEFT, customerTableScan, ordersTableScan, - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + return p.join(JoinType.LEFT, customerTableScan, ordersTableScan, + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -1051,7 +1053,7 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.INNER, customerTableScan, ordersTableScan, emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable, ordersCustKeyVariable), Optional.empty()); + return p.join(JoinType.INNER, customerTableScan, ordersTableScan, emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable, ordersCustKeyVariable), Optional.empty()); }) .matches(expectedLogicalProperties); @@ -1080,7 +1082,7 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.INNER, p.limit(11, customerTableScan), p.limit(12, ordersTableScan), emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable, ordersCustKeyVariable), Optional.empty()); + return p.join(JoinType.INNER, p.limit(11, customerTableScan), p.limit(12, ordersTableScan), emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable, ordersCustKeyVariable), Optional.empty()); }) .matches(expectedLogicalProperties); @@ -1119,7 +1121,7 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), orderTableConstraints); - return p.join(JoinNode.Type.INNER, customerTableScan, ordersTableScan, emptyList(), + return p.join(JoinType.INNER, customerTableScan, ordersTableScan, emptyList(), ImmutableList.of(customerCustKeyVariable, customerCommentVariable, ordersOrderKeyVariable, ordersCustKeyVariable, ordersCommentVariable), Optional.empty()); }) @@ -1138,7 +1140,7 @@ public void testJoinNodeLogicalProperties() .on(p -> { ValuesNode values1 = p.values(1, c1); ValuesNode values2 = p.values(1, c2); - return p.join(JoinNode.Type.INNER, values1, values2, emptyList(), ImmutableList.of(c1, c2), Optional.empty()); + return p.join(JoinType.INNER, values1, values2, emptyList(), ImmutableList.of(c1, c2), Optional.empty()); }) .matches(expectedLogicalProperties); @@ -1155,7 +1157,7 @@ public void testJoinNodeLogicalProperties() .on(p -> { ValuesNode values1 = p.values(1, c3); ValuesNode values2 = p.values(1, c4); - return p.join(JoinNode.Type.FULL, values1, values2, emptyList(), ImmutableList.of(c3, c4), Optional.empty()); + return p.join(JoinType.FULL, values1, values2, emptyList(), ImmutableList.of(c3, c4), Optional.empty()); }) .matches(expectedLogicalProperties); @@ -1183,7 +1185,7 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.FULL, customerTableScan, ordersTableScan, emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable), Optional.empty()); + return p.join(JoinType.FULL, customerTableScan, ordersTableScan, emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable), Optional.empty()); }) .matches(expectedLogicalProperties); @@ -1212,8 +1214,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.FULL, p.limit(12, customerTableScan), p.limit(10, ordersTableScan), - ImmutableList.of(new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)), + return p.join(JoinType.FULL, p.limit(12, customerTableScan), p.limit(10, ordersTableScan), + ImmutableList.of(new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable), Optional.empty()); }) @@ -1244,7 +1246,7 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.INNER, p.limit(2, customerTableScan), ordersTableScan, emptyList(), + return p.join(JoinType.INNER, p.limit(2, customerTableScan), ordersTableScan, emptyList(), ImmutableList.of(customerCustKeyVariable, ordersOrderKeyVariable, ordersCustKeyVariable), Optional.empty()); }) @@ -1275,7 +1277,7 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.INNER, ordersTableScan, p.limit(2, customerTableScan), emptyList(), + return p.join(JoinType.INNER, ordersTableScan, p.limit(2, customerTableScan), emptyList(), ImmutableList.of(ordersOrderKeyVariable, ordersCustKeyVariable, customerCustKeyVariable), Optional.empty()); }) @@ -1303,8 +1305,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(customerTableHandle)); - return p.join(JoinNode.Type.INNER, values, customerTableScan, - new JoinNode.EquiJoinClause(finalC1, customerCustKeyVariable)); + return p.join(JoinType.INNER, values, customerTableScan, + new EquiJoinClause(finalC1, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -1330,8 +1332,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(customerTableHandle)); - return p.join(JoinNode.Type.INNER, customerTableScan, values, - new JoinNode.EquiJoinClause(customerCustKeyVariable, finalC)); + return p.join(JoinType.INNER, customerTableScan, values, + new EquiJoinClause(customerCustKeyVariable, finalC)); }) .matches(expectedLogicalProperties); @@ -1354,8 +1356,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(customerTableHandle)); - return p.join(JoinNode.Type.FULL, customerTableScan, values, - new JoinNode.EquiJoinClause(customerCustKeyVariable, finalC2)); + return p.join(JoinType.FULL, customerTableScan, values, + new EquiJoinClause(customerCustKeyVariable, finalC2)); }) .matches(expectedLogicalProperties); @@ -1377,8 +1379,8 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(customerTableHandle)); - return p.join(JoinNode.Type.FULL, values, customerTableScan, - new JoinNode.EquiJoinClause(finalC3, customerCustKeyVariable)); + return p.join(JoinType.FULL, values, customerTableScan, + new EquiJoinClause(finalC3, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -1422,15 +1424,15 @@ public void testJoinNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(lineitemTableHandle)); - JoinNode customerOrderJoin = p.join(JoinNode.Type.INNER, + JoinNode customerOrderJoin = p.join(JoinType.INNER, customerTableScan, p.limit(6, ordersTableScan), - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); - return p.join(JoinNode.Type.INNER, + return p.join(JoinType.INNER, customerOrderJoin, lineitemTableScan, - new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderkeyVariable)); + new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderkeyVariable)); }) .matches(expectedLogicalProperties); } @@ -1692,19 +1694,19 @@ public void testAggregationNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(lineitemTableHandle)); - JoinNode customerOrderJoin = p.join(JoinNode.Type.INNER, + JoinNode customerOrderJoin = p.join(JoinType.INNER, customerTableScan, p.limit(6, ordersTableScan), - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); p.variable(lineitemExtendedPriceVariable); return p.aggregation(builder -> builder .addAggregation(p.variable("sum_price", DOUBLE), p.rowExpression("sum(l_extendedprice)")) .singleGroupingSet(lineitemLinenumberVariable, shipPriorityVariable) - .source(p.join(JoinNode.Type.INNER, + .source(p.join(JoinType.INNER, customerOrderJoin, lineitemTableScan, - new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderkeyVariable)))); + new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderkeyVariable)))); }) .matches(expectedLogicalProperties); @@ -1744,19 +1746,19 @@ public void testAggregationNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(lineitemTableHandle)); - JoinNode customerOrderJoin = p.join(JoinNode.Type.INNER, + JoinNode customerOrderJoin = p.join(JoinType.INNER, customerTableScan, p.limit(6, ordersTableScan), - new JoinNode.EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); + new EquiJoinClause(customerCustKeyVariable, ordersCustKeyVariable)); p.variable(lineitemExtendedPriceVariable); return p.aggregation(builder -> builder .addAggregation(p.variable("sum_price", DOUBLE), p.rowExpression("sum(l_extendedprice)")) .singleGroupingSet(lineitemLinenumberVariable, ordersOrderKeyVariable, shipPriorityVariable) - .source(p.join(JoinNode.Type.INNER, + .source(p.join(JoinType.INNER, customerOrderJoin, lineitemTableScan, - new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderkeyVariable)))); + new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderkeyVariable)))); }) .matches(expectedLogicalProperties); } @@ -1939,8 +1941,8 @@ public void testLimitNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - JoinNode ordersCustomerJoin = p.join(JoinNode.Type.INNER, ordersTableScan, customerTableScan, - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + JoinNode ordersCustomerJoin = p.join(JoinType.INNER, ordersTableScan, customerTableScan, + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); return p.limit(6, ordersCustomerJoin); }) @@ -1970,8 +1972,8 @@ public void testLimitNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.INNER, p.limit(5, ordersTableScan), customerTableScan, - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + return p.join(JoinType.INNER, p.limit(5, ordersTableScan), customerTableScan, + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); @@ -2047,8 +2049,8 @@ public void testTopNNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - JoinNode ordersCustomerJoin = p.join(JoinNode.Type.INNER, ordersTableScan, customerTableScan, - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + JoinNode ordersCustomerJoin = p.join(JoinType.INNER, ordersTableScan, customerTableScan, + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); return p.topN(6, ImmutableList.of(ordersCustKeyVariable, ordersOrderKeyVariable), ordersCustomerJoin); @@ -2079,8 +2081,8 @@ public void testTopNNodeLogicalProperties() TupleDomain.none(), tester().getTableConstraints(ordersTableHandle)); - return p.join(JoinNode.Type.INNER, p.topN(5, ImmutableList.of(ordersCustKeyVariable), ordersTableScan), customerTableScan, - new JoinNode.EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + return p.join(JoinType.INNER, p.topN(5, ImmutableList.of(ordersCustKeyVariable), ordersTableScan), customerTableScan, + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); }) .matches(expectedLogicalProperties); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCrossJoinColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCrossJoinColumns.java index 684fd9aa0cb18..32c7d11642d8f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCrossJoinColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCrossJoinColumns.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -46,7 +46,7 @@ public void testLeftInputNotReferenced() strictProject( ImmutableMap.of("rightValue", PlanMatchPattern.expression("rightValue")), join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), Optional.empty(), strictProject( @@ -65,7 +65,7 @@ public void testRightInputNotReferenced() strictProject( ImmutableMap.of("leftValue", PlanMatchPattern.expression("leftValue")), join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), Optional.empty(), values(ImmutableList.of("leftValue")), @@ -94,7 +94,7 @@ private static PlanNode buildProjectedCrossJoin(PlanBuilder p, Predicate buildJoin(p, variable -> variable.getName().equals("leftValue"))) .matches( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("leftKey", "rightKey")), Optional.of("leftValue > 5"), values("leftKey", "leftKeyHash", "leftValue"), @@ -71,7 +72,7 @@ public void testCrossJoinDoesNotFire() VariableReferenceExpression leftValue = p.variable("leftValue"); VariableReferenceExpression rightValue = p.variable("rightValue"); return p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.values(leftValue), p.values(rightValue), ImmutableList.of(), @@ -93,10 +94,10 @@ private static PlanNode buildJoin(PlanBuilder p, Predicate outputs = ImmutableList.of(leftValue, rightValue); return p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.values(leftKey, leftKeyHash, leftValue), p.values(rightKey, rightKeyHash, rightValue), - ImmutableList.of(new JoinNode.EquiJoinClause(leftKey, rightKey)), + ImmutableList.of(new EquiJoinClause(leftKey, rightKey)), outputs.stream() .filter(joinOutputFilter) .collect(toImmutableList()), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java index 6023708201528..f5dd30e2e2973 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java @@ -14,12 +14,13 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -48,7 +49,7 @@ public void testNotAllOutputsReferenced() strictProject( ImmutableMap.of("rightValue", PlanMatchPattern.expression("rightValue")), join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("leftKey", "rightKey")), Optional.empty(), values(ImmutableList.of("leftKey", "leftValue")), @@ -74,7 +75,7 @@ public void testCrossJoinDoesNotFire() return p.project( Assignments.of(), p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.values(leftValue), p.values(rightValue), ImmutableList.of(), @@ -99,10 +100,10 @@ private static PlanNode buildProjectedJoin(PlanBuilder p, Predicate p.aggregation(ab -> ab .source( p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), p.values(p.variable("COL2")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -77,8 +78,8 @@ public void testPushesAggregationThroughLeftJoin() project(ImmutableMap.of( "COL1", expression("COL1"), "COALESCE", expression("coalesce(AVG, AVG_NULL)")), - join(JoinNode.Type.INNER, ImmutableList.of(), - join(JoinNode.Type.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), + join(JoinType.INNER, ImmutableList.of(), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), values(ImmutableMap.of("COL1", 0)), aggregation( singleGroupingSet("COL2"), @@ -103,12 +104,12 @@ public void testPushesAggregationThroughLeftJoinWithOrderByFromRightSideColumn() .on(p -> p.aggregation(ab -> ab .source( p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.values( ImmutableList.of(p.variable("COL1"), p.variable("COL3")), ImmutableList.of(constantExpressions(BIGINT, 10L, 20L))), p.values(p.variable("COL2"), p.variable("COL4")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -126,8 +127,8 @@ public void testPushesAggregationThroughLeftJoinWithOrderByFromRightSideColumn() "COL1", expression("COL1"), "COL3", expression("COL3"), "COALESCE", expression("coalesce(AVG, AVG_NULL)")), - join(JoinNode.Type.INNER, ImmutableList.of(), - join(JoinNode.Type.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), + join(JoinType.INNER, ImmutableList.of(), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), values(ImmutableMap.of("COL1", 0, "COL3", 0)), aggregation( singleGroupingSet("COL2"), @@ -159,10 +160,10 @@ public void testPushesAggregationThroughRightJoin() tester().assertThat(new PushAggregationThroughOuterJoin(getFunctionManager())) .on(p -> p.aggregation(ab -> ab .source(p.join( - JoinNode.Type.RIGHT, + JoinType.RIGHT, p.values(p.variable("COL2")), p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL2"), p.variable("COL1"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL2"), p.variable("COL1"))), ImmutableList.of(p.variable("COL2"), p.variable("COL1")), Optional.empty(), Optional.empty(), @@ -173,8 +174,8 @@ public void testPushesAggregationThroughRightJoin() project(ImmutableMap.of( "COALESCE", expression("coalesce(AVG, AVG_NULL)"), "COL1", expression("COL1")), - join(JoinNode.Type.INNER, ImmutableList.of(), - join(JoinNode.Type.RIGHT, ImmutableList.of(equiJoinClause("COL2", "COL1")), + join(JoinType.INNER, ImmutableList.of(), + join(JoinType.RIGHT, ImmutableList.of(equiJoinClause("COL2", "COL1")), aggregation( singleGroupingSet("COL2"), ImmutableMap.of(Optional.of("AVG"), functionCall("avg", ImmutableList.of("COL2"))), @@ -199,12 +200,12 @@ public void testDoesNotFireWhenNotDistinct() tester().assertThat(new PushAggregationThroughOuterJoin(getFunctionManager())) .on(p -> p.aggregation(ab -> ab .source(p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.values( ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values(p.variable("COL2")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -218,7 +219,7 @@ public void testDoesNotFireWhenNotDistinct() .on(p -> p.aggregation(ab -> ab .source( p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.project(identityAssignments(p.variable("COL1", BIGINT)), p.aggregation(builder -> builder.singleGroupingSet(p.variable("COL1"), p.variable("unused")) @@ -227,7 +228,7 @@ public void testDoesNotFireWhenNotDistinct() ImmutableList.of(p.variable("COL1"), p.variable("unused")), ImmutableList.of(constantExpressions(BIGINT, 10L, 1L), constantExpressions(BIGINT, 10L, 2L)))))), p.values(p.variable("COL2")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -242,10 +243,10 @@ public void testDoesNotFireWhenGroupingOnInner() { tester().assertThat(new PushAggregationThroughOuterJoin(getFunctionManager())) .on(p -> p.aggregation(ab -> ab - .source(p.join(JoinNode.Type.LEFT, + .source(p.join(JoinType.LEFT, p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), p.values(p.variable("COL2"), p.variable("COL3")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -261,10 +262,10 @@ public void testDoesNotFireWhenAggregationDoesNotHaveSymbols() tester().assertThat(new PushAggregationThroughOuterJoin(getFunctionManager())) .on(p -> p.aggregation(ab -> ab .source(p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), p.values(ImmutableList.of(p.variable("COL2")), ImmutableList.of(constantExpressions(BIGINT, 20L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoinWithDefaultsForCorrelatedAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoinWithDefaultsForCorrelatedAggregations.java index 55005d62cc753..d0f3613fa90bd 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoinWithDefaultsForCorrelatedAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoinWithDefaultsForCorrelatedAggregations.java @@ -14,10 +14,11 @@ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.Ordering; import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -51,10 +52,10 @@ public void testPushesAggregationThroughLeftJoin() .on(p -> p.aggregation(ab -> ab .source( p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), p.values(p.variable("COL2")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -65,7 +66,7 @@ public void testPushesAggregationThroughLeftJoin() project(ImmutableMap.of( "COL1", expression("COL1"), "COALESCE", expression("coalesce(AVG, NULL)")), - join(JoinNode.Type.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), values(ImmutableMap.of("COL1", 0)), aggregation( singleGroupingSet("COL2"), @@ -83,12 +84,12 @@ public void testPushesAggregationThroughLeftJoinWithOrderByFromRightSideColumn() .on(p -> p.aggregation(ab -> ab .source( p.join( - JoinNode.Type.LEFT, + JoinType.LEFT, p.values( ImmutableList.of(p.variable("COL1"), p.variable("COL3")), ImmutableList.of(constantExpressions(BIGINT, 10L, 20L))), p.values(p.variable("COL2"), p.variable("COL4")), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), Optional.empty(), @@ -106,7 +107,7 @@ public void testPushesAggregationThroughLeftJoinWithOrderByFromRightSideColumn() "COL1", expression("COL1"), "COL3", expression("COL3"), "COALESCE", expression("coalesce(AVG, NULL)")), - join(JoinNode.Type.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("COL1", "COL2")), values(ImmutableMap.of("COL1", 0, "COL3", 0)), aggregation( singleGroupingSet("COL2"), @@ -127,10 +128,10 @@ public void testPushesAggregationThroughRightJoin() tester().assertThat(new PushAggregationThroughOuterJoin(getFunctionManager())) .on(p -> p.aggregation(ab -> ab .source(p.join( - JoinNode.Type.RIGHT, + JoinType.RIGHT, p.values(p.variable("COL2")), p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL2"), p.variable("COL1"))), + ImmutableList.of(new EquiJoinClause(p.variable("COL2"), p.variable("COL1"))), ImmutableList.of(p.variable("COL2"), p.variable("COL1")), Optional.empty(), Optional.empty(), @@ -141,7 +142,7 @@ public void testPushesAggregationThroughRightJoin() project(ImmutableMap.of( "COALESCE", expression("coalesce(AVG, NULL)"), "COL1", expression("COL1")), - join(JoinNode.Type.RIGHT, ImmutableList.of(equiJoinClause("COL2", "COL1")), + join(JoinType.RIGHT, ImmutableList.of(equiJoinClause("COL2", "COL1")), aggregation( singleGroupingSet("COL2"), ImmutableMap.of(Optional.of("AVG"), functionCall("avg", ImmutableList.of("COL2"))), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushDownFilterExpressionEvaluationThroughCrossJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushDownFilterExpressionEvaluationThroughCrossJoin.java index 9a2d70ba0b0ad..ab8014042f6ea 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushDownFilterExpressionEvaluationThroughCrossJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushDownFilterExpressionEvaluationThroughCrossJoin.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -44,7 +44,7 @@ public void testTriggerWithAddition() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("left_k1+left_k2 = right_k1+right_k2"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }) @@ -53,7 +53,7 @@ public void testTriggerWithAddition() filter( "add_1 = add_0", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), project( ImmutableMap.of("add_1", expression("left_k1+left_k2")), @@ -75,7 +75,7 @@ public void testNotTriggerWithAddition() p.variable("right_k2", BIGINT); return p.filter( p.rowExpression("left_k1+right_k1 = left_k2+right_k2"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1"), p.variable("left_k2")), p.values(p.variable("right_k1"), p.variable("right_k2")))); }).doesNotFire(); @@ -91,7 +91,7 @@ public void testTriggerWithArrayCardinality() p.variable("right_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("left_k1 = cardinality(right_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1")), p.values(p.variable("right_k1", new ArrayType(BIGINT))))); }) @@ -100,7 +100,7 @@ public void testTriggerWithArrayCardinality() filter( "left_k1 = card", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), values("left_k1"), project( @@ -120,7 +120,7 @@ public void testCast() p.variable("right_k2", VARCHAR); return p.filter( p.rowExpression("left_k1 = right_k1 or CAST(left_k2 AS BIGINT) = CAST(right_k2 AS BIGINT)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR), p.variable("left_k2", VARCHAR)), p.values(p.variable("right_k1", VARCHAR), p.variable("right_k2", VARCHAR)))); }) @@ -129,7 +129,7 @@ public void testCast() filter( "left_k1 = right_k1 OR cast_1 = cast_0", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), project( ImmutableMap.of("cast_1", expression("CAST(left_k2 AS bigint)")), @@ -151,7 +151,7 @@ public void testCoalesce() p.variable("right_k3", VARCHAR); return p.filter( p.rowExpression("left_k1 = right_k1 or CAST(left_k1 AS VARCHAR) = COALESCE(right_k2, right_k3)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1")), p.values(p.variable("right_k1"), p.variable("right_k2", VARCHAR), p.variable("right_k3", VARCHAR)))); }) @@ -160,7 +160,7 @@ public void testCoalesce() filter( "left_k1 = right_k1 OR cast_1 = expr", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), project( ImmutableMap.of("cast_1", expression("CAST(left_k1 AS varchar)")), @@ -180,7 +180,7 @@ public void testTriggerWithArrayContains() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(right_array_k1, cast(left_k1 as BIGINT))"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }) @@ -189,7 +189,7 @@ public void testTriggerWithArrayContains() filter( "contains(right_array_k1, cast_l)", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), project( ImmutableMap.of("cast_l", expression("CAST(left_k1 AS bigint)")), @@ -207,7 +207,7 @@ public void testTriggerWithArrayContains2() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(cast(right_array_k1 as array), left_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }) @@ -216,7 +216,7 @@ public void testTriggerWithArrayContains2() filter( "contains(cast_arr, left_k1)", join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(), values("left_k1"), project( @@ -234,7 +234,7 @@ public void testNotTriggerWithArrayContains() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(right_array_k1, cast(left_k1 as BIGINT)) or cast(left_k1 as BIGINT) > 2"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", VARCHAR)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }).doesNotFire(); @@ -250,7 +250,7 @@ public void testNotTriggerWithArrayContains2() p.variable("right_array_k1", new ArrayType(BIGINT)); return p.filter( p.rowExpression("contains(cast(right_array_k1 as array), left_k1)"), - p.join(JoinNode.Type.INNER, + p.join(JoinType.INNER, p.values(p.variable("left_k1", DOUBLE)), p.values(p.variable("right_array_k1", new ArrayType(BIGINT))))); }).doesNotFire(); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughOuterJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughOuterJoin.java index f8b884cdab1cf..8f27535cff1ed 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughOuterJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughOuterJoin.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; +import static com.facebook.presto.spi.plan.JoinType.FULL; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.limit; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; public class TestPushLimitThroughOuterJoin extends BaseRuleTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java index 61d84cdf641d1..ef888485017fb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -25,6 +25,7 @@ import static com.facebook.presto.SystemSessionProperties.PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; @@ -32,7 +33,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestPushPartialAggregationThroughJoin extends BaseRuleTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRedundantAggregateDistinctRemoval.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRedundantAggregateDistinctRemoval.java index 39612030ed601..bd7720db91ebf 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRedundantAggregateDistinctRemoval.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRedundantAggregateDistinctRemoval.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TestTableConstraintsConnectorFactory; @@ -20,7 +21,6 @@ import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesProviderImpl; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.FunctionCall; import com.google.common.collect.ImmutableList; @@ -142,7 +142,7 @@ public void complexQueryTests() .matches(output(anyTree( aggregation( aggregations, - join(JoinNode.Type.INNER, + join(JoinType.INNER, ImmutableList.of(equiJoinClause("custkey", "custkey_0")), tableScan("orders", ImmutableMap.of("totalprice", "totalprice", "orderkey", "orderkey", "custkey", "custkey")), tableScan("customer", ImmutableMap.of("custkey_0", "custkey"))))))); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantCastToVarcharInJoinClause.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantCastToVarcharInJoinClause.java index 46bb69beb7954..dc8b1850eadbd 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantCastToVarcharInJoinClause.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantCastToVarcharInJoinClause.java @@ -13,9 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -47,16 +48,16 @@ public void testCastBigIntToVarchar() VariableReferenceExpression leftCast = p.variable("left_cast", VARCHAR); VariableReferenceExpression rightCast = p.variable("right_cast", VARCHAR); return p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.project(assignment(leftCast, p.rowExpression("cast(left_bigint as varchar)")), p.values(leftBigint)), p.project(assignment(rightCast, p.rowExpression("cast(right_bigint as varchar)")), p.values(rightBigint)), - new JoinNode.EquiJoinClause(leftCast, rightCast)); + new EquiJoinClause(leftCast, rightCast)); }) .matches( join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("new_left", "new_right")), project( ImmutableMap.of("new_left", expression("left_bigint")), @@ -77,12 +78,12 @@ public void testCastIntToBigint() VariableReferenceExpression leftCast = p.variable("left_cast", BIGINT); VariableReferenceExpression rightCast = p.variable("right_cast", BIGINT); return p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.project(assignment(leftCast, p.rowExpression("cast(left_bigint as varchar)")), p.values(leftBigint)), p.project(assignment(rightCast, p.rowExpression("cast(right_bigint as varchar)")), p.values(rightBigint)), - new JoinNode.EquiJoinClause(leftCast, rightCast)); + new EquiJoinClause(leftCast, rightCast)); }).doesNotFire(); } @@ -96,11 +97,11 @@ public void testNoCast() VariableReferenceExpression leftCast = p.variable("left_cast", VARCHAR); VariableReferenceExpression rightCast = p.variable("right_cast", VARCHAR); return p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.values(leftCast), p.project(assignment(rightCast, p.rowExpression("cast(right_bigint as varchar)")), p.values(rightBigint)), - new JoinNode.EquiJoinClause(leftCast, rightCast)); + new EquiJoinClause(leftCast, rightCast)); }).doesNotFire(); } @@ -115,12 +116,12 @@ public void testCastDoubleToVarchar() VariableReferenceExpression leftCast = p.variable("left_cast", VARCHAR); VariableReferenceExpression rightCast = p.variable("right_cast", VARCHAR); return p.join( - JoinNode.Type.INNER, + JoinType.INNER, p.project(assignment(leftCast, p.rowExpression("cast(left_bigint as varchar)")), p.values(leftBigint)), p.project(assignment(rightCast, p.rowExpression("cast(right_bigint as varchar)")), p.values(rightBigint)), - new JoinNode.EquiJoinClause(leftCast, rightCast)); + new EquiJoinClause(leftCast, rightCast)); }).doesNotFire(); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java index 79fdcfb55a04e..9b4091800865c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java @@ -18,6 +18,7 @@ import com.facebook.presto.cost.CostComparator; import com.facebook.presto.cost.PlanNodeStatsEstimate; import com.facebook.presto.cost.VariableStatsEstimate; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -27,7 +28,6 @@ import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; @@ -51,6 +51,9 @@ import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.metadata.FunctionAndTypeManager.qualifyObjectName; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.AUTOMATIC; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -61,9 +64,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.variable; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRuntimeReorderJoinSides.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRuntimeReorderJoinSides.java index 4c284ae5a1258..84b75ed2d3e16 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRuntimeReorderJoinSides.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRuntimeReorderJoinSides.java @@ -20,12 +20,12 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.tpch.TpchColumnHandle; import com.facebook.presto.tpch.TpchTableHandle; @@ -41,14 +41,14 @@ import java.util.Optional; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; @Test(singleThreaded = true) public class TestRuntimeReorderJoinSides @@ -127,7 +127,7 @@ public void testDoesNotFireWhenNonTableScanUnderJoin() INNER, p.tableScan(nationTableHandle, ImmutableList.of(p.variable("nationkey", BIGINT)), ImmutableMap.of(p.variable("nationkey", BIGINT), nationColumnHandle)), p.values(new PlanNodeId("valuesB"), 10000, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkey", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkey", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("nationkey", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) .doesNotFire(); @@ -151,7 +151,7 @@ public void testDoesNotFireWithoutBasicStatistics() .addSource(suppNode) .addInputsSet(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT))) .fixedHashDistributionPartitioningScheme(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableList.of(p.variable("nationkeyS", BIGINT)))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), ImmutableList.of(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), Optional.empty()); }) @@ -182,7 +182,7 @@ public void testDoesNotFireWhenProbeSideLarger() .addSource(suppNode) .addInputsSet(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT))) .fixedHashDistributionPartitioningScheme(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableList.of(p.variable("nationkeyS", BIGINT)))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), ImmutableList.of(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), Optional.empty()); }) @@ -216,7 +216,7 @@ public void testDoesNotFireWhenSwappedJoinInvalid() .addSource(suppNode) .addInputsSet(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT))) .fixedHashDistributionPartitioningScheme(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableList.of(p.variable("nationkeyS", BIGINT)))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), ImmutableList.of(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), Optional.empty(), Optional.empty(), @@ -251,7 +251,7 @@ public void testFlipsAndAdjustExchangeWhenProbeSideSmaller() .addSource(suppNode) .addInputsSet(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT))) .fixedHashDistributionPartitioningScheme(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableList.of(p.variable("nationkeyS", BIGINT)))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), ImmutableList.of(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), Optional.empty(), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarAggregationToJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarAggregationToJoin.java index f6ad83dd973b0..9c46094851533 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarAggregationToJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarAggregationToJoin.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -89,7 +89,7 @@ public void rewritesOnSubqueryWithoutProjection() .matches( project(ImmutableMap.of("sum_1", expression("sum_1"), "corr", expression("corr")), aggregation(ImmutableMap.of("sum_1", functionCall("sum", ImmutableList.of("a"))), - join(JoinNode.Type.LEFT, + join(JoinType.LEFT, ImmutableList.of(), assignUniqueId("unique", values(ImmutableMap.of("corr", 0))), @@ -113,7 +113,7 @@ public void rewritesOnSubqueryWithProjection() .matches( project(ImmutableMap.of("corr", expression("corr"), "expr", expression("(\"sum_1\" + 1)")), aggregation(ImmutableMap.of("sum_1", functionCall("sum", ImmutableList.of("a"))), - join(JoinNode.Type.LEFT, + join(JoinType.LEFT, ImmutableList.of(), assignUniqueId("unique", values(ImmutableMap.of("corr", 0))), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java index b818a6a0df911..b938d0478a0b0 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToLeftEarlyOutJoin.java @@ -15,6 +15,7 @@ import com.facebook.presto.cost.PlanNodeStatsEstimate; import com.facebook.presto.cost.VariableStatsEstimate; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -22,7 +23,6 @@ import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -37,6 +37,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; @@ -45,7 +46,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.relational.Expressions.variable; import static java.util.Collections.emptyList; @@ -77,7 +77,7 @@ public void testAggregationPushedDown() p.assignUniqueId(unique, p.values(new PlanNodeId("valuesA"), 1000, a)), p.values(new PlanNodeId("valuesB"), 100, b), - ImmutableList.of(new JoinNode.EquiJoinClause(a, b)), + ImmutableList.of(new EquiJoinClause(a, b)), ImmutableList.of(unique, a), Optional.empty()))); }) @@ -118,7 +118,7 @@ public void testAggregationPushedDown() p.assignUniqueId(unique, p.values(new PlanNodeId("valuesA"), 1000, a)), p.values(new PlanNodeId("valuesBC"), 100, b, c), - ImmutableList.of(new JoinNode.EquiJoinClause(a, b)), + ImmutableList.of(new EquiJoinClause(a, b)), ImmutableList.of(unique, a, c), Optional.empty()))); }) @@ -153,7 +153,7 @@ public void testFeatureDisabled() p.values(new PlanNodeId("valuesB"), b), p.assignUniqueId(unique, p.values(new PlanNodeId("valuesA"), a)), - new JoinNode.EquiJoinClause(b, a))))); + new EquiJoinClause(b, a))))); }; tester().assertThat(new TransformDistinctInnerJoinToLeftEarlyOutJoin()) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java index eb6f7f5ce5cc2..9c8b7f8a736b9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformDistinctInnerJoinToRightEarlyOutJoin.java @@ -15,6 +15,7 @@ import com.facebook.presto.cost.PlanNodeStatsEstimate; import com.facebook.presto.cost.VariableStatsEstimate; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -22,7 +23,6 @@ import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -36,6 +36,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; @@ -43,7 +44,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.relational.Expressions.variable; import static java.util.Collections.emptyList; @@ -75,7 +75,7 @@ public void testAggregationPushedDown() p.values(new PlanNodeId("valuesB"), 100, b), p.assignUniqueId(unique, p.values(new PlanNodeId("valuesA"), 1000, a)), - new JoinNode.EquiJoinClause(b, a)))); + new EquiJoinClause(b, a)))); }) .overrideStats("valuesA", PlanNodeStatsEstimate.builder() .setOutputRowCount(1000) @@ -115,7 +115,7 @@ public void testFeatureDisabled() p.values(new PlanNodeId("valuesB"), b), p.assignUniqueId(unique, p.values(new PlanNodeId("valuesA"), a)), - new JoinNode.EquiJoinClause(b, a))))); + new EquiJoinClause(b, a))))); }; tester().assertThat(new TransformDistinctInnerJoinToRightEarlyOutJoin()) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java index 1de6e1807f5ef..5ffd45cc8320c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin.java @@ -32,6 +32,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.AUTOMATIC; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -43,7 +44,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.relational.Expressions.inSubquery; import static java.util.Collections.emptyList; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedLateralToJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedLateralToJoin.java index 54c6fa57e62e4..c8d37119b1642 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedLateralToJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedLateralToJoin.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -31,7 +31,7 @@ public void test() tester() .assertThat(new TransformUncorrelatedLateralToJoin()) .on(p -> p.lateral(emptyList(), p.values(), p.values())) - .matches(join(JoinNode.Type.INNER, emptyList(), values(), values())); + .matches(join(JoinType.INNER, emptyList(), values(), values())); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index c44e5a2b2e2c8..c077d7d7cfd19 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -32,9 +32,12 @@ import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.MarkDistinctNode; import com.facebook.presto.spi.plan.Ordering; @@ -758,17 +761,17 @@ protected ExchangeNode build() } } - public JoinNode join(JoinNode.Type joinType, PlanNode left, PlanNode right, JoinNode.EquiJoinClause... criteria) + public JoinNode join(JoinType joinType, PlanNode left, PlanNode right, EquiJoinClause... criteria) { return join(joinType, left, right, Optional.empty(), criteria); } - public JoinNode join(JoinNode.Type joinType, PlanNode left, PlanNode right, RowExpression filter, JoinNode.EquiJoinClause... criteria) + public JoinNode join(JoinType joinType, PlanNode left, PlanNode right, RowExpression filter, EquiJoinClause... criteria) { return join(joinType, left, right, Optional.of(filter), criteria); } - private JoinNode join(JoinNode.Type joinType, PlanNode left, PlanNode right, Optional filter, JoinNode.EquiJoinClause... criteria) + private JoinNode join(JoinType joinType, PlanNode left, PlanNode right, Optional filter, EquiJoinClause... criteria) { return join( joinType, @@ -784,16 +787,16 @@ private JoinNode join(JoinNode.Type joinType, PlanNode left, PlanNode right, Opt Optional.empty()); } - public JoinNode join(JoinNode.Type type, PlanNode left, PlanNode right, List criteria, List outputVariables, Optional filter) + public JoinNode join(JoinType type, PlanNode left, PlanNode right, List criteria, List outputVariables, Optional filter) { return join(type, left, right, criteria, outputVariables, filter, Optional.empty(), Optional.empty()); } public JoinNode join( - JoinNode.Type type, + JoinType type, PlanNode left, PlanNode right, - List criteria, + List criteria, List outputVariables, Optional filter, Optional leftHashVariable, @@ -803,10 +806,10 @@ public JoinNode join( } public JoinNode join( - JoinNode.Type type, + JoinType type, PlanNode left, PlanNode right, - List criteria, + List criteria, List outputVariables, Optional filter, Optional leftHashVariable, @@ -817,15 +820,15 @@ public JoinNode join( } public JoinNode join( - JoinNode.Type type, + JoinType type, PlanNode left, PlanNode right, - List criteria, + List criteria, List outputVariables, Optional filter, Optional leftHashVariable, Optional rightHashVariable, - Optional distributionType, + Optional distributionType, Map dynamicFilters) { return new JoinNode(Optional.empty(), idAllocator.getNextId(), type, left, right, criteria, outputVariables, filter, leftHashVariable, rightHashVariable, distributionType, dynamicFilters); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlans.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlans.java index 6b3db9c5a6454..aca3982dfad43 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlans.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlans.java @@ -42,6 +42,7 @@ import static com.facebook.presto.SystemSessionProperties.USE_STREAMING_EXCHANGE_FOR_MARK_DISTINCT; import static com.facebook.presto.execution.QueryManagerConfig.ExchangeMaterializationStrategy.ALL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.PARTITIONED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.analyzer.FeaturesConfig.PartitioningPrecisionStrategy.PREFER_EXACT_PARTITIONING; @@ -62,7 +63,6 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_MATERIALIZED; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static org.testng.Assert.assertEquals; /** diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateCrossJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateCrossJoins.java index 0a4aced6c6c86..63a0e47eeb56f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateCrossJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateCrossJoins.java @@ -22,12 +22,12 @@ import java.util.Optional; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestEliminateCrossJoins extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java index 4623e9c13347d..82e180b0d78e3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java @@ -22,6 +22,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.JoinType.FULL; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.PARTITIONED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; @@ -36,7 +37,6 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; public class TestFullOuterJoinWithCoalesce extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestHashGenerationOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestHashGenerationOptimizer.java index 2351c215a3a50..8205255352159 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestHashGenerationOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestHashGenerationOptimizer.java @@ -22,6 +22,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.SystemSessionProperties.SKIP_HASH_GENERATION_FOR_JOIN_WITH_TABLE_SCAN_INPUT; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -32,7 +33,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPLICATE; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestHashGenerationOptimizer extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java index dfa2eb4a01e1b..ef7c94781f0cc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.common.block.SortOrder; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; @@ -22,7 +23,6 @@ import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.iterative.rule.GatherAndMergeWindows; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantIdentityProjections; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.tree.FrameBound; import com.facebook.presto.sql.tree.WindowFrame; @@ -466,7 +466,7 @@ public void testNotMergeAcrossJoinBranches() assertUnitPlan(sql, anyTree( filter("SUM = AVG", - join(JoinNode.Type.INNER, ImmutableList.of(), + join(JoinType.INNER, ImmutableList.of(), any( window(windowMatcherBuilder -> windowMatcherBuilder .specification(leftSpecification) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRandomizeNullKeyInOuterJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRandomizeNullKeyInOuterJoin.java index 02bb163661cb8..34515421dfb69 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRandomizeNullKeyInOuterJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRandomizeNullKeyInOuterJoin.java @@ -23,6 +23,9 @@ import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_HASH_GENERATION; import static com.facebook.presto.SystemSessionProperties.RANDOMIZE_OUTER_JOIN_NULL_KEY; import static com.facebook.presto.SystemSessionProperties.RANDOMIZE_OUTER_JOIN_NULL_KEY_STRATEGY; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; @@ -30,9 +33,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; public class TestRandomizeNullKeyInOuterJoin extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveRedundantDistinctAggregation.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveRedundantDistinctAggregation.java index 16ca76afeeb27..76e11d2240d4c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveRedundantDistinctAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveRedundantDistinctAggregation.java @@ -23,6 +23,7 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; @@ -33,7 +34,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestRemoveRedundantDistinctAggregation extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java index 65e6aa975deb1..53b660309c125 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestRemoveUnsupportedDynamicFilters.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; @@ -47,6 +48,7 @@ import java.util.Optional; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.ExpressionUtils.combineDisjuncts; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; @@ -54,7 +56,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.optimizations.PredicatePushDown.createDynamicFilterExpression; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -103,7 +104,7 @@ public void testUnconsumedDynamicFilterInJoin() INNER, builder.filter(builder.rowExpression("ORDERS_OK > 0"), ordersTableScanNode), lineitemTableScanNode, - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), @@ -129,7 +130,7 @@ public void testDynamicFilterConsumedOnBuildSide() builder.filter( createDynamicFilterExpression("DF", ordersOrderKeyVariable, metadata.getFunctionAndTypeManager()), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), @@ -156,7 +157,7 @@ public void testUnmatchedDynamicFilter() builder.rowExpression("LINEITEM_OK > 0"), createDynamicFilterExpression("DF", lineitemOrderKeyVariable, metadata.getFunctionAndTypeManager())), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(), Optional.empty(), Optional.empty(), @@ -190,7 +191,7 @@ public void testNestedDynamicFilterDisjunctionRewrite() builder.rowExpression("LINEITEM_OK IS NOT NULL"), createDynamicFilterExpression("DF", lineitemOrderKeyVariable, metadata.getFunctionAndTypeManager()))), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), @@ -221,7 +222,7 @@ public void testNestedDynamicFilterConjunctionRewrite() builder.rowExpression("LINEITEM_OK IS NOT NULL"), createDynamicFilterExpression("DF", lineitemOrderKeyVariable, metadata.getFunctionAndTypeManager()))), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java index d703b64fbd18f..5769413e42f6b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java @@ -14,11 +14,11 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; @@ -104,7 +104,7 @@ public void testInnerJoin() project( ImmutableMap.of("expr_14", expression("'3-MEDIUM'")), join( - JoinNode.Type.INNER, + JoinType.INNER, ImmutableList.of(equiJoinClause("orderkey", "orderkey_0")), anyTree( tableScan("lineitem", ImmutableMap.of("orderkey", "orderkey", "tax", "tax"))), @@ -122,7 +122,7 @@ public void testLeftJoinNotTrigger() output( ImmutableList.of("orderkey_0", "expr_9", "tax"), join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("orderkey", "orderkey_0")), anyTree( tableScan("lineitem", ImmutableMap.of("orderkey", "orderkey", "tax", "tax"))), @@ -144,7 +144,7 @@ public void testLeftJoinTrigger() project( ImmutableMap.of("expr_26", expression("'O'")), join( - JoinNode.Type.LEFT, + JoinType.LEFT, ImmutableList.of(equiJoinClause("orderkey", "orderkey_0")), anyTree( filter( @@ -216,7 +216,7 @@ public void testJoinWithFilter() project( semiJoin("partkey", "suppkey_17", "expr_37", project( - join(JoinNode.Type.INNER, + join(JoinType.INNER, ImmutableList.of(), anyTree( filter("orderkey = 10", @@ -444,7 +444,7 @@ public void testJoinPlanChange() VariableReferenceExpression key2 = planBuilder.variable("key2", INTEGER); VariableReferenceExpression count = planBuilder.variable("cnt"); return planBuilder.join( - JoinNode.Type.INNER, + JoinType.INNER, planBuilder.filter( planBuilder.rowExpression("key1= 3"), planBuilder.values(key1)), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestDynamicFiltersChecker.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestDynamicFiltersChecker.java index 45578ed348db4..cbde8ce2a5e3f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestDynamicFiltersChecker.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestDynamicFiltersChecker.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; @@ -26,7 +27,6 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.testing.TestingTransactionHandle; @@ -41,8 +41,8 @@ import java.util.Optional; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.optimizations.PredicatePushDown.createDynamicFilterExpression; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestDynamicFiltersChecker extends BasePlanTest @@ -90,7 +90,7 @@ public void testUnconsumedDynamicFilterInJoin() INNER, builder.filter(builder.rowExpression("ORDERS_OK > 0"), ordersTableScanNode), lineitemTableScanNode, - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), @@ -110,7 +110,7 @@ public void testDynamicFilterConsumedOnBuildSide() builder.filter( createDynamicFilterExpression("DF", ordersOrderKeyVariable, metadata.getFunctionAndTypeManager()), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), @@ -133,7 +133,7 @@ public void testUnmatchedDynamicFilter() builder.rowExpression("LINEITEM_OK > 0"), createDynamicFilterExpression("DF", lineitemOrderKeyVariable, metadata.getFunctionAndTypeManager())), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), @@ -160,7 +160,7 @@ public void testUnmatchedNestedDynamicFilter() builder.rowExpression("LINEITEM_OK IS NOT NULL"), createDynamicFilterExpression("DF", lineitemOrderKeyVariable, metadata.getFunctionAndTypeManager()))), lineitemTableScanNode), - ImmutableList.of(new JoinNode.EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), + ImmutableList.of(new EquiJoinClause(ordersOrderKeyVariable, lineitemOrderKeyVariable)), ImmutableList.of(ordersOrderKeyVariable), Optional.empty(), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java index 0ea6e16e54d67..dda3ae05da8ed 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java @@ -42,10 +42,10 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.groupingSets; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestValidateAggregationsWithDefaultValues extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java index c886f5303858f..ac950270e99c6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.sql.parser.SqlParser; @@ -27,7 +28,6 @@ import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.tpch.TpchColumnHandle; import com.facebook.presto.tpch.TpchTableHandle; @@ -43,7 +43,7 @@ import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.BigintType.BIGINT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; public class TestValidateStreamingJoins @@ -109,7 +109,7 @@ public void testValidateSuccessful() .addSource(p.tableScan(supplierTableHandle, ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableMap.of(p.variable("nationkeyS", BIGINT), nationColumnHandle, p.variable("suppkey", BIGINT), suppColumnHandle))) .addInputsSet(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT))) .fixedHashDistributionPartitioningScheme(ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableList.of(p.variable("nationkeyS", BIGINT)))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), ImmutableList.of(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), Optional.empty())); } @@ -122,7 +122,7 @@ public void testValidateFailed() INNER, p.tableScan(nationTableHandle, ImmutableList.of(p.variable("nationkeyN", BIGINT)), ImmutableMap.of(p.variable("nationkeyN", BIGINT), nationColumnHandle)), p.tableScan(supplierTableHandle, ImmutableList.of(p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), ImmutableMap.of(p.variable("nationkeyS", BIGINT), nationColumnHandle, p.variable("suppkey", BIGINT), suppColumnHandle)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT))), ImmutableList.of(p.variable("nationkeyN", BIGINT), p.variable("nationkeyS", BIGINT), p.variable("suppkey", BIGINT)), Optional.empty())); } diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java b/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java index 00a3554da6e98..10e341acbe595 100644 --- a/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java +++ b/presto-main/src/test/java/com/facebook/presto/util/TestGraphvizPrinter.java @@ -18,6 +18,8 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; @@ -153,7 +155,7 @@ public void testPrintLogicalForJoinNode() PlanNode node = new JoinNode( Optional.empty(), new PlanNodeId("join"), - JoinNode.Type.INNER, + JoinType.INNER, TEST_TABLE_SCAN_NODE, //Left : Probe side valuesNode, //Right : Build side Collections.emptyList(), //No Criteria @@ -164,7 +166,7 @@ public void testPrintLogicalForJoinNode() Optional.empty(), //NO filter Optional.empty(), Optional.empty(), - Optional.of(JoinNode.DistributionType.REPLICATED), + Optional.of(JoinDistributionType.REPLICATED), ImmutableMap.of()); String actual = printLogical( diff --git a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java index 53c296aa3aaa9..4ed89d14704a0 100644 --- a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java +++ b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/geospatial/TestSpatialJoinPlanning.java @@ -20,10 +20,10 @@ import com.facebook.presto.plugin.memory.MemoryConnectorFactory; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.tpch.TpchConnectorFactory; import com.google.common.collect.ImmutableList; @@ -368,7 +368,7 @@ public void testNotContains() "WHERE NOT ST_Contains(ST_GeometryFromText(wkt), ST_Point(lng, lat))", anyTree( filter("NOT ST_Contains(ST_GeometryFromText(cast(wkt as varchar)), ST_Point(lng, lat))", - join(JoinNode.Type.INNER, emptyList(), + join(JoinType.INNER, emptyList(), anyTree(values(ImmutableMap.of("lng", 0, "lat", 1))), values(ImmutableMap.of("wkt", 0)))))); } @@ -381,7 +381,7 @@ public void testNotIntersects() "WHERE NOT ST_Intersects(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt))", anyTree( filter("NOT ST_Intersects(ST_GeometryFromText(cast(wkt_a as varchar)), ST_GeometryFromText(cast(wkt_b as varchar)))", - join(JoinNode.Type.INNER, emptyList(), + join(JoinType.INNER, emptyList(), anyTree(values(ImmutableMap.of("wkt_a", 0, "name_a", 1))), values(ImmutableMap.of("wkt_b", 0, "name_b", 1)))))); } @@ -393,7 +393,7 @@ public void testContainsWithEquiClause() "FROM " + POINTS_SQL + ", " + POLYGONS_SQL + " " + "WHERE a.name = b.name AND ST_Contains(ST_GeometryFromText(wkt), ST_Point(lng, lat))", anyTree( - join(JoinNode.Type.INNER, ImmutableList.of(equiJoinClause("name_a", "name_b")), + join(JoinType.INNER, ImmutableList.of(equiJoinClause("name_a", "name_b")), Optional.of("ST_Contains(ST_GeometryFromText(cast(wkt as varchar)), ST_Point(lng, lat))"), anyTree(values(ImmutableMap.of("lng", 0, "lat", 1, "name_a", 2))), anyTree(values(ImmutableMap.of("wkt", 0, "name_b", 1)))))); @@ -406,7 +406,7 @@ public void testIntersectsWithEquiClause() "FROM (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a')) AS a (wkt, name), (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a')) AS b (wkt, name) " + "WHERE a.name = b.name AND ST_Intersects(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt))", anyTree( - join(JoinNode.Type.INNER, ImmutableList.of(equiJoinClause("name_a", "name_b")), + join(JoinType.INNER, ImmutableList.of(equiJoinClause("name_a", "name_b")), Optional.of("ST_Intersects(ST_GeometryFromText(cast(wkt_a as varchar)), ST_GeometryFromText(cast(wkt_B as varchar)))"), anyTree(values(ImmutableMap.of("wkt_a", 0, "name_a", 1))), anyTree(values(ImmutableMap.of("wkt_b", 0, "name_b", 1)))))); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/PickJoinSides.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/PickJoinSides.java index 8f8112fe06e80..e1413709c6ea5 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/PickJoinSides.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/optimizers/PickJoinSides.java @@ -41,12 +41,12 @@ import static com.facebook.presto.SystemSessionProperties.isSizeBasedJoinDistributionTypeEnabled; import static com.facebook.presto.spark.PrestoSparkSessionProperties.isAdaptiveJoinSideSwitchingEnabled; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.createRuntimeSwappedJoinNode; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.isBelowBroadcastLimit; import static com.facebook.presto.sql.planner.iterative.rule.JoinSwappingUtils.isSmallerThanThreshold; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static java.util.Objects.requireNonNull; diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/TestPrestoSparkHistoryBasedTracking.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/TestPrestoSparkHistoryBasedTracking.java index 27c25e3567906..1d6fe05cb98f6 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/TestPrestoSparkHistoryBasedTracking.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/TestPrestoSparkHistoryBasedTracking.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.plan.JoinDistributionType; import com.facebook.presto.spi.plan.OutputNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; @@ -94,7 +95,7 @@ public void testBroadcastJoin() "(SELECT * FROM test_orders where ds = '2020-09-02' and substr(CAST(custkey AS VARCHAR), 1, 3) = '370') t2 ON t1.orderkey = t2.orderkey", getSession()); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) - .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinNode.DistributionType.PARTITIONED)) + .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinDistributionType.PARTITIONED)) .findFirst() .isPresent()); @@ -108,7 +109,7 @@ public void testBroadcastJoin() "(SELECT * FROM test_orders where ds = '2020-09-02' and substr(CAST(custkey AS VARCHAR), 1, 3) = '370') t2 ON t1.orderkey = t2.orderkey", getSession()); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) - .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinNode.DistributionType.REPLICATED)) + .where(node -> node instanceof JoinNode && ((JoinNode) node).getDistributionType().get().equals(JoinDistributionType.REPLICATED)) .findFirst() .isPresent()); } diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestIterativePlanFragmenter.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestIterativePlanFragmenter.java index 97476ec87714b..ffbef957a4f0f 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestIterativePlanFragmenter.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestIterativePlanFragmenter.java @@ -49,6 +49,9 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinDistributionType; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -200,7 +203,7 @@ public void testIterativePlanFragmenter() JoinNode join = join("join", remoteExchange1, localExchange, - JoinNode.DistributionType.PARTITIONED, + JoinDistributionType.PARTITIONED, "orderkey_1", "orderkey_0"); Map types = ImmutableMap.of( @@ -336,19 +339,19 @@ private AggregationNode aggregation(String id, PlanNode source) * EquiJoinClause is created from symbols in form of: * symbol[0] = symbol[1] AND symbol[2] = symbol[3] AND ... */ - private JoinNode join(String planNodeId, PlanNode left, PlanNode right, JoinNode.DistributionType distributionType, String... symbols) + private JoinNode join(String planNodeId, PlanNode left, PlanNode right, JoinDistributionType distributionType, String... symbols) { checkArgument(symbols.length % 2 == 0); - ImmutableList.Builder criteria = ImmutableList.builder(); + ImmutableList.Builder criteria = ImmutableList.builder(); for (int i = 0; i < symbols.length; i += 2) { - criteria.add(new JoinNode.EquiJoinClause(new VariableReferenceExpression(Optional.empty(), symbols[i], BIGINT), new VariableReferenceExpression(Optional.empty(), symbols[i + 1], BIGINT))); + criteria.add(new EquiJoinClause(new VariableReferenceExpression(Optional.empty(), symbols[i], BIGINT), new VariableReferenceExpression(Optional.empty(), symbols[i + 1], BIGINT))); } return new JoinNode( Optional.empty(), new PlanNodeId(planNodeId), - JoinNode.Type.INNER, + JoinType.INNER, left, right, criteria.build(), diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkPhysicalResourceAllocationStrategy.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkPhysicalResourceAllocationStrategy.java index f437239e85026..6b204e7c9d1fc 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkPhysicalResourceAllocationStrategy.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkPhysicalResourceAllocationStrategy.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; @@ -31,7 +32,6 @@ import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.TableStatistics; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.testing.TestingMetadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -112,7 +112,7 @@ private PlanNode getPlanToTest(Session session, Metadata metadata) VariableReferenceExpression filteringSource = planBuilder.variable("filteringSource"); TableScanNode b = planBuilder.tableScan(ImmutableList.of(filteringSource), ImmutableMap.of(filteringSource, new TestingMetadata.TestingColumnHandle("filteringSource"))); - return planBuilder.join(JoinNode.Type.LEFT, a, b); + return planBuilder.join(JoinType.LEFT, a, b); } private PhysicalResourceSettings getSettingsHolder(Session session, Metadata metadata) diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/optimizers/TestPickJoinSides.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/optimizers/TestPickJoinSides.java index a8d7d9dee31e2..1fd1314a46c08 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/optimizers/TestPickJoinSides.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/optimizers/TestPickJoinSides.java @@ -33,12 +33,13 @@ import com.facebook.presto.cost.VariableStatsEstimate; import com.facebook.presto.spark.PrestoSparkSessionProperties; import com.facebook.presto.spark.PrestoSparkSessionPropertyManagerProvider; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.facebook.presto.tpch.TpchConnectorFactory; import com.google.common.collect.ImmutableList; @@ -57,6 +58,11 @@ import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.spark.PrestoSparkSessionProperties.ADAPTIVE_JOIN_SIDE_SWITCHING_ENABLED; +import static com.facebook.presto.spi.plan.JoinDistributionType.PARTITIONED; +import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; @@ -64,11 +70,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.remoteSource; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; -import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; import static com.facebook.presto.sql.planner.plan.JoinNode.flipType; import static com.facebook.presto.testing.TestngUtils.toDataProvider; @@ -100,12 +101,12 @@ public void tearDown() @DataProvider(name = "joinTypes") public static Object[][] joinTypes() { - return Arrays.stream(JoinNode.Type.values()) + return Arrays.stream(JoinType.values()) .collect(toDataProvider()); } @Test(dataProvider = "joinTypes") - public void testFlipsWhenProbeSmaller(JoinNode.Type joinType) + public void testFlipsWhenProbeSmaller(JoinType joinType) { int aSize = 100; int bSize = 10_000; @@ -127,7 +128,7 @@ public void testFlipsWhenProbeSmaller(JoinNode.Type joinType) new PlanNodeId("valuesB"), ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), Optional.empty(), @@ -144,7 +145,7 @@ public void testFlipsWhenProbeSmaller(JoinNode.Type joinType) } @Test(dataProvider = "joinTypes") - public void testDoesNotFireWhenTablesSameSize(JoinNode.Type joinType) + public void testDoesNotFireWhenTablesSameSize(JoinType joinType) { int aSize = 100; int bSize = 100; @@ -164,7 +165,7 @@ public void testDoesNotFireWhenTablesSameSize(JoinNode.Type joinType) p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), Optional.empty(), @@ -175,7 +176,7 @@ public void testDoesNotFireWhenTablesSameSize(JoinNode.Type joinType) } @Test(dataProvider = "joinTypes") - public void testFlipWhenOneTableMuchSmallerAndJoinCardinalityUnknown(JoinNode.Type joinType) + public void testFlipWhenOneTableMuchSmallerAndJoinCardinalityUnknown(JoinType joinType) { int aRows = 100; int bRows = 10_000; @@ -195,7 +196,7 @@ public void testFlipWhenOneTableMuchSmallerAndJoinCardinalityUnknown(JoinNode.Ty joinType, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), Optional.empty(), @@ -238,7 +239,7 @@ public void testFlipsWhenSourceIsSmall() LEFT, p.filter(new PlanNodeId("filterB"), TRUE_CONSTANT, p.values(new PlanNodeId("valuesB"), bRows, b1)), p.values(new PlanNodeId("valuesA"), aRows, a1), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty(), Optional.empty(), @@ -295,7 +296,7 @@ public void testFlipWhenSizeDifferenceLarge() new PlanNodeId("filterB"), TRUE_CONSTANT, p.remoteSource(new PlanNodeId("remoteSourceB"), ImmutableList.of(new PlanFragmentId(2)), ImmutableList.of(b1))), - ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), + ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty(), Optional.empty(), @@ -324,7 +325,7 @@ public void testFlipWhenSizeDifferenceLarge() .fixedHashDistributionPartitioningScheme(ImmutableList.of(a1), ImmutableList.of(a1)) .addInputsSet(a1) .addSource(p.remoteSource(new PlanNodeId("remoteSourceA"), ImmutableList.of(new PlanFragmentId(1)), ImmutableList.of(a1)))), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty(), Optional.empty(), @@ -363,7 +364,7 @@ public void testFlipWhenSizeDifferenceLarge() TRUE_CONSTANT, p.remoteSource(new PlanNodeId("remoteSourceB"), ImmutableList.of(new PlanFragmentId(2)), ImmutableList.of(b1))), p.remoteSource(new PlanNodeId("remoteSourceA"), ImmutableList.of(new PlanFragmentId(1)), ImmutableList.of(a1)), - ImmutableList.of(new JoinNode.EquiJoinClause(b1, a1)), + ImmutableList.of(new EquiJoinClause(b1, a1)), ImmutableList.of(b1, a1), Optional.empty(), Optional.empty(), @@ -395,7 +396,7 @@ public void testDoesNotFireWhenDisabled() p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), Optional.empty(), @@ -426,7 +427,7 @@ public void testDoesNotFireForReplicatedJoin() p.values( ImmutableList.of(p.variable("B1")), ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), - ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), + ImmutableList.of(new EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), Optional.empty(), diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/ConnectorJoinNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/ConnectorJoinNode.java new file mode 100644 index 0000000000000..0d0efb015ca4e --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/ConnectorJoinNode.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public class ConnectorJoinNode + extends PlanNode +{ + private final List sources; + private final JoinType type; + private final Set criteria; + private final Set filters; + private final Optional distributionType; + private final List outputVariables; + + public ConnectorJoinNode( + PlanNodeId id, + List sources, + Optional statsEquivalentPlanNode, + JoinType type, + Set criteria, + Set filters, + Optional distributionType, + List outputVariables) + { + super(Optional.empty(), id, statsEquivalentPlanNode); + this.sources = requireNonNull(sources, "sources is null"); + this.type = requireNonNull(type, "type is null"); + this.criteria = requireNonNull(criteria, "criteria is null"); + this.filters = requireNonNull(filters, "filters is null"); + this.distributionType = requireNonNull(distributionType, "distributionType is null"); + this.outputVariables = requireNonNull(outputVariables, "outputVariables is null"); + } + + @Override + public List getSources() + { + return sources; + } + + public JoinType getType() + { + return type; + } + + public Set getCriteria() + { + return criteria; + } + + public Set getFilters() + { + return filters; + } + + public Optional getDistributionType() + { + return distributionType; + } + + @Override + public List getOutputVariables() + { + return outputVariables; + } + + @Override + public PlanNode replaceChildren(List newChildren) + { + return new ConnectorJoinNode(getId(), newChildren, getStatsEquivalentPlanNode(), type, criteria, filters, distributionType, outputVariables); + } + + @Override + public PlanNode assignStatsEquivalentPlanNode(Optional statsEquivalentPlanNode) + { + return new ConnectorJoinNode(getId(), getSources(), getStatsEquivalentPlanNode(), getType(), getCriteria(), getFilters(), getDistributionType(), outputVariables); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConnectorJoinNode that = (ConnectorJoinNode) o; + return Objects.equals(sources, that.sources) && + Objects.equals(type, that.type) && + Objects.equals(criteria, that.criteria) && + Objects.equals(filters, that.filters) && + Objects.equals(outputVariables, that.outputVariables); + } + + @Override + public int hashCode() + { + return Objects.hash(sources, type, criteria, filters, outputVariables); + } + + @Override + public String toString() + { + return "ConnectorJoinNode{" + + "sources=" + sources + + ", type=" + type + + ", criteria=" + criteria + + ", filters=" + filters + + ", outputVariables=" + outputVariables + + '}'; + } + + @Override + public R accept(PlanVisitor visitor, C context) + { + return visitor.visitConnectorJoinNode(this, context); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java new file mode 100644 index 0000000000000..ae01d9b912b3e --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/EquiJoinClause.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class EquiJoinClause +{ + private final VariableReferenceExpression left; + private final VariableReferenceExpression right; + + @JsonCreator + public EquiJoinClause(@JsonProperty("left") VariableReferenceExpression left, @JsonProperty("right") VariableReferenceExpression right) + { + this.left = requireNonNull(left, "left is null"); + this.right = requireNonNull(right, "right is null"); + } + + @JsonProperty + public VariableReferenceExpression getLeft() + { + return left; + } + + @JsonProperty + public VariableReferenceExpression getRight() + { + return right; + } + + public EquiJoinClause flip() + { + return new EquiJoinClause(right, left); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + + if (obj == null || !this.getClass().equals(obj.getClass())) { + return false; + } + + EquiJoinClause other = (EquiJoinClause) obj; + + return Objects.equals(this.left, other.left) && + Objects.equals(this.right, other.right); + } + + @Override + public int hashCode() + { + return Objects.hash(left, right); + } + + @Override + public String toString() + { + return format("%s = %s", left, right); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinDistributionType.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinDistributionType.java new file mode 100644 index 0000000000000..8e742740b61de --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinDistributionType.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +public enum JoinDistributionType +{ + PARTITIONED, + REPLICATED, +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinType.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinType.java new file mode 100644 index 0000000000000..5146189ede139 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/JoinType.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import java.util.List; + +public enum JoinType +{ + INNER("InnerJoin"), + LEFT("LeftJoin"), + RIGHT("RightJoin"), + FULL("FullJoin"); + + private final String joinLabel; + + JoinType(String joinLabel) + { + this.joinLabel = joinLabel; + } + + public String getJoinLabel() + { + return joinLabel; + } + + public boolean mustPartition() + { + // With REPLICATED, the unmatched rows from right-side would be duplicated. + return this == RIGHT || this == FULL; + } + + public boolean mustReplicate(List criteria) + { + // There is nothing to partition on + return criteria.isEmpty() && (this == INNER || this == LEFT); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java index 1ee77f1e988b8..daf2cebd5673c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java @@ -104,4 +104,9 @@ public R visitSequence(SequenceNode node, C context) { return visitPlan(node, context); } + + public R visitConnectorJoinNode(ConnectorJoinNode node, C context) + { + return visitPlan(node, context); + } }