From 3d9ec1c6cbf9ea499b5e18caf20cd462e247f033 Mon Sep 17 00:00:00 2001 From: Shixuan Fan Date: Mon, 15 Apr 2019 16:21:17 -0700 Subject: [PATCH 01/66] Remove redundant cast --- .../src/main/java/com/facebook/presto/hive/HiveMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index ddc71a5a83514..82c70e92d9c53 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -407,7 +407,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect Iterable> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint, session).getPartitions()) .map(hivePartition -> - (List) IntStream.range(0, partitionColumns.size()) + IntStream.range(0, partitionColumns.size()) .mapToObj(fieldIdToColumnHandle::get) .map(columnHandle -> hivePartition.getKeys().get(columnHandle).getValue()) .collect(toList())) From 1f00cf22e858bebb20ce0fcca37991cf6e239de8 Mon Sep 17 00:00:00 2001 From: Shixuan Fan Date: Mon, 15 Apr 2019 16:11:08 -0700 Subject: [PATCH 02/66] Mark unpartitioned table in HiveWrittenPartitions We already use "" for unpartitioned table input, and this commit would unify the representation of unpartitioned table in output as well. --- .../facebook/presto/hive/HiveMetadata.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 82c70e92d9c53..e71702e532e2e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -135,6 +135,7 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; +import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; import static com.facebook.presto.hive.HivePartitionManager.extractPartitionValues; import static com.facebook.presto.hive.HiveSessionProperties.getHiveStorageFormat; import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableSchema; @@ -1191,25 +1192,27 @@ public Optional finishCreateTable(ConnectorSession sess metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics); - if (!handle.getPartitionedBy().isEmpty()) { - if (isRespectTableFormat(session)) { - Verify.verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat()); - } - for (PartitionUpdate update : partitionUpdates) { - Partition partition = buildPartitionObject(session, table, update); - PartitionStatistics partitionStatistics = createPartitionStatistics( - session, - update.getStatistics(), - columnTypes, - getColumnStatistics(partitionComputedStatistics, partition.getValues())); - metastore.addPartition( - session, - handle.getSchemaName(), - handle.getTableName(), - buildPartitionObject(session, table, update), - update.getWritePath(), - partitionStatistics); - } + if (handle.getPartitionedBy().isEmpty()) { + return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID))); + } + + if (isRespectTableFormat(session)) { + Verify.verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat()); + } + for (PartitionUpdate update : partitionUpdates) { + Partition partition = buildPartitionObject(session, table, update); + PartitionStatistics partitionStatistics = createPartitionStatistics( + session, + update.getStatistics(), + columnTypes, + getColumnStatistics(partitionComputedStatistics, partition.getValues())); + metastore.addPartition( + session, + handle.getSchemaName(), + handle.getTableName(), + buildPartitionObject(session, table, update), + update.getWritePath(), + partitionStatistics); } return Optional.of(new HiveWrittenPartitions( @@ -1489,6 +1492,7 @@ else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode return Optional.of(new HiveWrittenPartitions( partitionUpdates.stream() .map(PartitionUpdate::getName) + .map(name -> name.isEmpty() ? UNPARTITIONED_ID : name) .collect(Collectors.toList()))); } From 95cbe760e15985289b0cd901b91b0aa8f66cfac2 Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:41 -0400 Subject: [PATCH 03/66] Fix incorrect condition in ExchangeNode --- .../java/com/facebook/presto/sql/planner/plan/ExchangeNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java index d192e871c8498..550f9788aee2e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java @@ -120,7 +120,7 @@ public ExchangeNode( "local exchanges do not support constant partition function arguments"); checkArgument(!scope.isRemote() || type == REPARTITION || !partitioningScheme.isReplicateNullsAndAny(), "Only REPARTITION can replicate remotely"); - checkArgument(scope != REMOTE_MATERIALIZED || type != REPARTITION, "Only REPARTITION can be REMOTE_MATERIALIZED: %s", type); + checkArgument(scope != REMOTE_MATERIALIZED || type == REPARTITION, "Only REPARTITION can be REMOTE_MATERIALIZED: %s", type); orderingScheme.ifPresent(ordering -> { PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); From e6052c901bb312a714d68ff52a2966092b58ad8f Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:43 -0400 Subject: [PATCH 04/66] Add DiscardingOutputBuffer --- .../buffer/DiscardingOutputBuffer.java | 183 ++++++++++++++++++ .../execution/buffer/LazyOutputBuffer.java | 3 + .../execution/buffer/OutputBuffers.java | 8 + 3 files changed, 194 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java new file mode 100644 index 0000000000000..2a92c45abaed4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/DiscardingOutputBuffer.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution.buffer; + +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.StateMachine; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; +import io.airlift.units.DataSize; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import static com.facebook.presto.execution.buffer.BufferState.FAILED; +import static com.facebook.presto.execution.buffer.BufferState.FINISHED; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.util.Objects.requireNonNull; + +public class DiscardingOutputBuffer + implements OutputBuffer +{ + private static final ListenableFuture NON_BLOCKED = immediateFuture(null); + + private final OutputBuffers outputBuffers; + private final StateMachine state; + + private final AtomicLong totalPagesAdded = new AtomicLong(); + private final AtomicLong totalRowsAdded = new AtomicLong(); + + public DiscardingOutputBuffer(OutputBuffers outputBuffers, StateMachine state) + { + this.outputBuffers = requireNonNull(outputBuffers, "outputBuffers is null"); + this.state = requireNonNull(state, "state is null"); + } + + @Override + public OutputBufferInfo getInfo() + { + BufferState state = this.state.get(); + + return new OutputBufferInfo( + "DISCARD", + state, + state.canAddBuffers(), + state.canAddPages(), + 0, + 0, + totalRowsAdded.get(), + totalPagesAdded.get(), + ImmutableList.of()); + } + + @Override + public boolean isFinished() + { + return state.get() == FINISHED; + } + + @Override + public double getUtilization() + { + return 0; + } + + @Override + public boolean isOverutilized() + { + return false; + } + + @Override + public void addStateChangeListener(StateMachine.StateChangeListener stateChangeListener) + { + state.addStateChangeListener(stateChangeListener); + } + + @Override + public void setOutputBuffers(OutputBuffers newOutputBuffers) + { + requireNonNull(newOutputBuffers, "newOutputBuffers is null"); + + // ignore buffers added after query finishes, which can happen when a query is canceled + // also ignore old versions, which is normal + if (state.get().isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) { + return; + } + + // no more buffers can be added but verify this is valid state change + outputBuffers.checkValidTransition(newOutputBuffers); + } + + @Override + public ListenableFuture get(OutputBuffers.OutputBufferId bufferId, long token, DataSize maxSize) + { + throw new UnsupportedOperationException("DiscardingOutputBuffer must not have any active readers"); + } + + @Override + public void acknowledge(OutputBuffers.OutputBufferId bufferId, long token) + { + throw new UnsupportedOperationException("DiscardingOutputBuffer must not have any active readers"); + } + + @Override + public void abort(OutputBuffers.OutputBufferId bufferId) {} + + @Override + public ListenableFuture isFull() + { + return NON_BLOCKED; + } + + @Override + public void enqueue(Lifespan lifespan, List pages) + { + // update stats + long rowCount = pages.stream().mapToLong(SerializedPage::getPositionCount).sum(); + totalRowsAdded.addAndGet(rowCount); + totalPagesAdded.addAndGet(pages.size()); + } + + @Override + public void enqueue(Lifespan lifespan, int partition, List pages) + { + checkState(partition == 0, "Expected partition number to be zero"); + enqueue(lifespan, pages); + } + + @Override + public void setNoMorePages() + { + state.set(FINISHED); + } + + @Override + public void destroy() + { + state.setIf(FINISHED, oldState -> !oldState.isTerminal()); + } + + @Override + public void fail() + { + state.setIf(FAILED, oldState -> !oldState.isTerminal()); + } + + @Override + public void setNoMorePagesForLifespan(Lifespan lifespan) + { + // NOOP + } + + @Override + public void registerLifespanCompletionCallback(Consumer callback) + { + // NOOP + } + + @Override + public boolean isFinishedForLifespan(Lifespan lifespan) + { + return true; + } + + @Override + public long getPeakMemoryUsage() + { + return 0; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java index 55dfd515f1cc1..a0c0e48e02238 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java @@ -168,6 +168,9 @@ public void setOutputBuffers(OutputBuffers newOutputBuffers) case ARBITRARY: delegate = new ArbitraryOutputBuffer(taskInstanceId, state, maxBufferSize, systemMemoryContextSupplier, executor); break; + case DISCARDING: + delegate = new DiscardingOutputBuffer(newOutputBuffers, state); + break; } // process pending aborts and reads outside of synchronized lock diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java index 8d13ab4702da6..7a7b73472197f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/OutputBuffers.java @@ -26,6 +26,7 @@ import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.ARBITRARY; import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.BROADCAST; +import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.DISCARDING; import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION; @@ -38,6 +39,7 @@ public final class OutputBuffers { public static final int BROADCAST_PARTITION_ID = 0; + private static final OutputBuffers DISCARDING_OUTPUT_BUFFERS = createInitialEmptyOutputBuffers(DISCARDING).withNoMoreBufferIds(); public static OutputBuffers createInitialEmptyOutputBuffers(BufferType type) { @@ -59,11 +61,17 @@ else if (partitioningHandle.equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new OutputBuffers(type, 0, false, ImmutableMap.of()); } + public static OutputBuffers createDiscardingOutputBuffers() + { + return DISCARDING_OUTPUT_BUFFERS; + } + public enum BufferType { PARTITIONED, BROADCAST, ARBITRARY, + DISCARDING, } private final BufferType type; From db4b9a1d1a5e80cc7bde23c3b9522c3cdefe5882 Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:44 -0400 Subject: [PATCH 05/66] Postpone temporary tables split enumeration --- .../presto/sql/planner/LazySplitSource.java | 94 +++++++++++++++++++ .../presto/sql/planner/PlanFragmenter.java | 6 +- .../sql/planner/SplitSourceFactory.java | 5 +- .../sql/planner/plan/TableScanNode.java | 28 +++++- 4 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java new file mode 100644 index 0000000000000..2b3fa2dbc68b7 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LazySplitSource.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.connector.ConnectorId; +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.spi.connector.ConnectorPartitionHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.split.SplitSource; +import com.google.common.util.concurrent.ListenableFuture; + +import javax.annotation.concurrent.GuardedBy; + +import java.util.function.Supplier; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class LazySplitSource + implements SplitSource +{ + private final Supplier supplier; + + @GuardedBy("this") + private SplitSource delegate; + @GuardedBy("this") + private boolean closed; + + public LazySplitSource(Supplier supplier) + { + this.supplier = requireNonNull(supplier, "supplier is null"); + } + + @Override + public ConnectorId getConnectorId() + { + return getDelegate().getConnectorId(); + } + + @Override + public ConnectorTransactionHandle getTransactionHandle() + { + return getDelegate().getTransactionHandle(); + } + + @Override + public ListenableFuture getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) + { + return getDelegate().getNextBatch(partitionHandle, lifespan, maxSize); + } + + @Override + public boolean isFinished() + { + return getDelegate().isFinished(); + } + + @Override + public synchronized void close() + { + // already closed + if (closed) { + return; + } + closed = true; + + // not yet initialized + if (delegate == null) { + return; + } + + delegate.close(); + } + + private synchronized SplitSource getDelegate() + { + checkState(!closed, "split source is closed"); + if (delegate == null) { + delegate = supplier.get(); + } + return delegate; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java index 9fde9e440083f..3194f1ab0ad35 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java @@ -560,7 +560,8 @@ private TableScanNode createTemporaryTableScan( assignments, Optional.of(layoutHandle), TupleDomain.all(), - TupleDomain.all()); + TupleDomain.all(), + true); } private TableFinishNode createTemporaryTableWrite( @@ -1074,7 +1075,8 @@ public PlanNode visitTableScan(TableScanNode node, RewriteContext context) node.getAssignments(), Optional.of(newTableLayoutHandle), node.getCurrentConstraint(), - node.getEnforcedConstraint()); + node.getEnforcedConstraint(), + node.isTemporaryTable()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java index 6499742adce99..edbaa4047faa5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java @@ -59,6 +59,7 @@ import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.GROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; @@ -122,11 +123,13 @@ public Map visitExplainAnalyze(ExplainAnalyzeNode node, public Map visitTableScan(TableScanNode node, Void context) { // get dataSource for table - SplitSource splitSource = splitSourceProvider.getSplits( + Supplier splitSourceSupplier = () -> splitSourceProvider.getSplits( session, node.getLayout().get(), stageExecutionDescriptor.isScanGroupedExecution(node.getId()) ? GROUPED_SCHEDULING : UNGROUPED_SCHEDULING); + SplitSource splitSource = node.isTemporaryTable() ? new LazySplitSource(splitSourceSupplier) : splitSourceSupplier.get(); + splitSources.add(splitSource); return ImmutableMap.of(node.getId(), splitSource); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java index 5e178e6e94fd2..5158d04701dae 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableScanNode.java @@ -50,13 +50,16 @@ public class TableScanNode private final TupleDomain enforcedConstraint; + private final boolean temporaryTable; + @JsonCreator public TableScanNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("table") TableHandle table, @JsonProperty("outputSymbols") List outputs, @JsonProperty("assignments") Map assignments, - @JsonProperty("layout") Optional tableLayout) + @JsonProperty("layout") Optional tableLayout, + @JsonProperty("temporaryTable") boolean temporaryTable) { // This constructor is for JSON deserialization only. Do not use. super(id); @@ -65,6 +68,7 @@ public TableScanNode( this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs"); this.tableLayout = requireNonNull(tableLayout, "tableLayout is null"); + this.temporaryTable = temporaryTable; this.currentConstraint = null; this.enforcedConstraint = null; } @@ -75,7 +79,7 @@ public TableScanNode( List outputs, Map assignments) { - this(id, table, outputs, assignments, Optional.empty(), TupleDomain.all(), TupleDomain.all()); + this(id, table, outputs, assignments, Optional.empty(), TupleDomain.all(), TupleDomain.all(), false); } public TableScanNode( @@ -86,6 +90,19 @@ public TableScanNode( Optional tableLayout, TupleDomain currentConstraint, TupleDomain enforcedConstraint) + { + this(id, table, outputs, assignments, tableLayout, currentConstraint, enforcedConstraint, false); + } + + public TableScanNode( + PlanNodeId id, + TableHandle table, + List outputs, + Map assignments, + Optional tableLayout, + TupleDomain currentConstraint, + TupleDomain enforcedConstraint, + boolean temporaryTable) { super(id); this.table = requireNonNull(table, "table is null"); @@ -95,6 +112,7 @@ public TableScanNode( this.tableLayout = requireNonNull(tableLayout, "tableLayout is null"); this.currentConstraint = requireNonNull(currentConstraint, "currentConstraint is null"); this.enforcedConstraint = requireNonNull(enforcedConstraint, "enforcedConstraint is null"); + this.temporaryTable = temporaryTable; if (!currentConstraint.isAll() || !enforcedConstraint.isAll()) { checkArgument(tableLayout.isPresent(), "tableLayout must be present when currentConstraint or enforcedConstraint is non-trivial"); } @@ -125,6 +143,12 @@ public Map getAssignments() return assignments; } + @JsonProperty + public boolean isTemporaryTable() + { + return temporaryTable; + } + /** * A TupleDomain that represents a predicate that every row this TableScan node * produces is guaranteed to satisfy. From e2608cdf87298d61dbe9cf45166a38689938d3cc Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:45 -0400 Subject: [PATCH 06/66] Support dependent stages in SqlQueryScheduler --- .../scheduler/AllAtOnceExecutionSchedule.java | 4 +- .../scheduler/SqlQueryScheduler.java | 304 +++++++++++++++--- .../facebook/presto/sql/planner/SubPlan.java | 6 - 3 files changed, 267 insertions(+), 47 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java index 6d16d4952525e..4a8ae95e22450 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java @@ -44,7 +44,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; @@ -98,10 +97,9 @@ static List getPreferredScheduleOrder(Collection f Set rootFragments = fragments.stream() .filter(fragment -> !remoteSources.contains(fragment.getId())) .collect(toImmutableSet()); - checkArgument(rootFragments.size() == 1, "Expected one root fragment, but found: " + rootFragments); Visitor visitor = new Visitor(fragments); - visitor.processFragment(getOnlyElement(rootFragments).getId()); + rootFragments.forEach(fragment -> visitor.processFragment(fragment.getId())); return visitor.getSchedulerOrder(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index f29be2a97af1c..09e6b7eae2a70 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -38,10 +38,12 @@ import com.facebook.presto.sql.planner.NodePartitionMap; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.PartitioningHandle; +import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.SplitSourceFactory; import com.facebook.presto.sql.planner.SubPlan; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.facebook.presto.sql.planner.plan.PlanNodeId; +import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -80,8 +82,10 @@ import static com.facebook.presto.execution.StageState.CANCELED; import static com.facebook.presto.execution.StageState.FAILED; import static com.facebook.presto.execution.StageState.FINISHED; +import static com.facebook.presto.execution.StageState.PLANNED; import static com.facebook.presto.execution.StageState.RUNNING; import static com.facebook.presto.execution.StageState.SCHEDULED; +import static com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers; import static com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; @@ -98,6 +102,8 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.newConcurrentHashSet; +import static com.google.common.collect.Streams.stream; +import static com.google.common.graph.Traverser.forTree; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static io.airlift.concurrent.MoreFutures.whenAnyComplete; @@ -114,6 +120,8 @@ public class SqlQueryScheduler { private final QueryStateMachine queryStateMachine; private final ExecutionPolicy executionPolicy; + private final SubPlan plan; + private final StreamingPlanSection sectionedPlan; private final Map stages; private final ExecutorService executor; private final StageId rootStageId; @@ -122,6 +130,7 @@ public class SqlQueryScheduler private final SplitSchedulerStats schedulerStats; private final boolean summarizeTaskInfo; private final AtomicBoolean started = new AtomicBoolean(); + private final AtomicBoolean scheduling = new AtomicBoolean(); public static SqlQueryScheduler createSqlQueryScheduler( QueryStateMachine queryStateMachine, @@ -184,6 +193,7 @@ private SqlQueryScheduler( SplitSchedulerStats schedulerStats) { this.queryStateMachine = requireNonNull(queryStateMachine, "queryStateMachine is null"); + this.plan = requireNonNull(plan, "plan is null"); this.executionPolicy = requireNonNull(executionPolicy, "schedulerPolicyFactory is null"); this.schedulerStats = requireNonNull(schedulerStats, "schedulerStats is null"); this.summarizeTaskInfo = summarizeTaskInfo; @@ -192,20 +202,19 @@ private SqlQueryScheduler( ImmutableMap.Builder stageSchedulers = ImmutableMap.builder(); ImmutableMap.Builder stageLinkages = ImmutableMap.builder(); - // Only fetch a distribution once per query to assure all stages see the same machine assignments - Map partitioningCache = new HashMap<>(); - OutputBufferId rootBufferId = Iterables.getOnlyElement(rootOutputBuffers.getBuffers().keySet()); + sectionedPlan = extractStreamingSections(plan); List stages = createStages( (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations), locationFactory, - plan.withBucketToPartition(Optional.of(new int[1])), + sectionedPlan, + Optional.of(new int[1]), + rootOutputBuffers, nodeScheduler, remoteTaskFactory, splitSourceFactory, session, splitBatchSize, - partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), nodePartitioningManager, queryExecutor, schedulerExecutor, @@ -214,9 +223,7 @@ private SqlQueryScheduler( stageSchedulers, stageLinkages); - SqlStageExecution rootStage = stages.get(0); - rootStage.setOutputBuffers(rootOutputBuffers); - this.rootStageId = rootStage.getStageId(); + this.rootStageId = stages.get(0).getStageId(); this.stages = stages.stream() .collect(toImmutableMap(SqlStageExecution::getStageId, identity())); @@ -253,6 +260,12 @@ else if (state == ABORTED) { // this should never happen, since abort can only be triggered in query clean up after the query is finished queryStateMachine.transitionToFailed(new PrestoException(GENERIC_INTERNAL_ERROR, "Query stage was aborted")); } + else if (state == FINISHED) { + List nextStagesToSchedule = getStagesReadyForExecution(); + if (!nextStagesToSchedule.isEmpty()) { + startScheduling(nextStagesToSchedule); + } + } else if (queryStateMachine.getQueryState() == QueryState.STARTING) { // if the stage has at least one task, we are running if (stage.hasTasks()) { @@ -289,9 +302,76 @@ private static URI getBufferLocation(RemoteTask remoteTask, OutputBufferId rootB } private List createStages( + ExchangeLocationsConsumer locationsConsumer, + LocationFactory locationFactory, + StreamingPlanSection section, + Optional bucketToPartition, + OutputBuffers outputBuffers, + NodeScheduler nodeScheduler, + RemoteTaskFactory remoteTaskFactory, + SplitSourceFactory splitSourceFactory, + Session session, + int splitBatchSize, + NodePartitioningManager nodePartitioningManager, + ExecutorService queryExecutor, + ScheduledExecutorService schedulerExecutor, + FailureDetector failureDetector, + NodeTaskMap nodeTaskMap, + ImmutableMap.Builder stageSchedulers, + ImmutableMap.Builder stageLinkages) + { + ImmutableList.Builder stages = ImmutableList.builder(); + + // Only fetch a distribution once per section to ensure all stages see the same machine assignments + Map partitioningCache = new HashMap<>(); + List sectionStages = createStreamingLinkedStages( + locationsConsumer, + locationFactory, + section.getPlan().withBucketToPartition(bucketToPartition), + nodeScheduler, + remoteTaskFactory, + splitSourceFactory, + session, + splitBatchSize, + partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), + nodePartitioningManager, + queryExecutor, + schedulerExecutor, + failureDetector, + nodeTaskMap, + stageSchedulers, + stageLinkages); + sectionStages.get(0).setOutputBuffers(outputBuffers); + stages.addAll(sectionStages); + + for (StreamingPlanSection childSection : section.getChildren()) { + stages.addAll(createStages( + discardingLocationConsumer(), + locationFactory, + childSection, + Optional.empty(), + createDiscardingOutputBuffers(), + nodeScheduler, + remoteTaskFactory, + splitSourceFactory, + session, + splitBatchSize, + nodePartitioningManager, + queryExecutor, + schedulerExecutor, + failureDetector, + nodeTaskMap, + stageSchedulers, + stageLinkages)); + } + + return stages.build(); + } + + private List createStreamingLinkedStages( ExchangeLocationsConsumer parent, LocationFactory locationFactory, - SubPlan plan, + StreamingSubPlan plan, NodeScheduler nodeScheduler, RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, @@ -309,7 +389,7 @@ private List createStages( ImmutableList.Builder stages = ImmutableList.builder(); PlanFragmentId fragmentId = plan.getFragment().getId(); - StageId stageId = new StageId(queryStateMachine.getQueryId(), fragmentId.getId()); + StageId stageId = getStageId(fragmentId); SqlStageExecution stage = createSqlStageExecution( stageId, locationFactory.createStageLocation(stageId), @@ -416,11 +496,11 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { } ImmutableSet.Builder childStagesBuilder = ImmutableSet.builder(); - for (SubPlan subStagePlan : plan.getChildren()) { - List subTree = createStages( + for (StreamingSubPlan stagePlan : plan.getChildren()) { + List subTree = createStreamingLinkedStages( stage::addExchangeLocations, locationFactory, - subStagePlan.withBucketToPartition(bucketToPartition), + stagePlan.withBucketToPartition(bucketToPartition), nodeScheduler, remoteTaskFactory, splitSourceFactory, @@ -489,29 +569,28 @@ public StageInfo getStageInfo() .map(SqlStageExecution::getStageInfo) .collect(toImmutableMap(StageInfo::getStageId, identity())); - return buildStageInfo(rootStageId, stageInfos); + return buildStageInfo(plan, stageInfos); } - private StageInfo buildStageInfo(StageId stageId, Map stageInfos) + private StageInfo buildStageInfo(SubPlan subPlan, Map stageInfos) { - StageInfo parent = stageInfos.get(stageId); - checkArgument(parent != null, "No stageInfo for %s", parent); - List childStages = stageLinkages.get(stageId).getChildStageIds().stream() - .map(childStageId -> buildStageInfo(childStageId, stageInfos)) - .collect(toImmutableList()); - if (childStages.isEmpty()) { - return parent; + StageInfo stageInfo = stageInfos.get(getStageId(subPlan.getFragment().getId())); + checkArgument(stageInfo != null, "No stageInfo for %s", stageInfo); + if (subPlan.getChildren().isEmpty()) { + return stageInfo; } return new StageInfo( - parent.getStageId(), - parent.getState(), - parent.getSelf(), - parent.getPlan(), - parent.getTypes(), - parent.getStageStats(), - parent.getTasks(), - childStages, - parent.getFailureCause()); + stageInfo.getStageId(), + stageInfo.getState(), + stageInfo.getSelf(), + stageInfo.getPlan(), + stageInfo.getTypes(), + stageInfo.getStageStats(), + stageInfo.getTasks(), + subPlan.getChildren().stream() + .map(plan -> buildStageInfo(plan, stageInfos)) + .collect(toImmutableList()), + stageInfo.getFailureCause()); } public long getUserMemoryReservation() @@ -539,15 +618,69 @@ public Duration getTotalCpuTime() public void start() { if (started.compareAndSet(false, true)) { - executor.submit(this::schedule); + startScheduling(getStagesReadyForExecution()); } } - private void schedule() + private List getStagesReadyForExecution() + { + return stream(forTree(StreamingPlanSection::getChildren).depthFirstPreOrder(sectionedPlan)) + // get all sections ready for execution + .filter(this::isReadyForExecution) + // get all stages in the sections + .flatMap(section -> stream(forTree(StreamingSubPlan::getChildren).depthFirstPreOrder(section.getPlan()))) + .map(StreamingSubPlan::getFragment) + .map(PlanFragment::getId) + .map(this::getStageExecution) + .collect(toImmutableList()); + } + + private boolean isReadyForExecution(StreamingPlanSection section) { + SqlStageExecution stage = getStageExecution(section.getPlan().getFragment().getId()); + if (stage.getState() != PLANNED) { + // already scheduled + return false; + } + for (StreamingPlanSection child : section.getChildren()) { + SqlStageExecution childRootStage = getStageExecution(child.getPlan().getFragment().getId()); + if (childRootStage.getState() != FINISHED) { + return false; + } + } + return true; + } + + private SqlStageExecution getStageExecution(PlanFragmentId planFragmentId) + { + return stages.get(getStageId(planFragmentId)); + } + + private StageId getStageId(PlanFragmentId fragmentId) + { + return new StageId(queryStateMachine.getQueryId(), fragmentId.getId()); + } + + private void startScheduling(Collection stages) + { + requireNonNull(stages); + // still scheduling the previous batch of stages + if (scheduling.get()) { + return; + } + executor.submit(() -> schedule(stages)); + } + + private void schedule(Collection stages) + { + if (!scheduling.compareAndSet(false, true)) { + // still scheduling the previous batch of stages + return; + } + try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) { Set completedStages = new HashSet<>(); - ExecutionSchedule executionSchedule = executionPolicy.createExecutionSchedule(stages.values()); + ExecutionSchedule executionSchedule = executionPolicy.createExecutionSchedule(stages); while (!executionSchedule.isFinished()) { List> blockedStages = new ArrayList<>(); for (SqlStageExecution stage : executionSchedule.getStagesToSchedule()) { @@ -591,7 +724,7 @@ else if (!result.getBlocked().isDone()) { } // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel) - for (SqlStageExecution stage : stages.values()) { + for (SqlStageExecution stage : stages) { if (!completedStages.contains(stage.getStageId()) && stage.getState().isDone()) { stageLinkages.get(stage.getStageId()) .processScheduleResults(stage.getState(), ImmutableSet.of()); @@ -610,22 +743,29 @@ else if (!result.getBlocked().isDone()) { } } - for (SqlStageExecution stage : stages.values()) { + for (SqlStageExecution stage : stages) { StageState state = stage.getState(); if (state != SCHEDULED && state != RUNNING && !state.isDone()) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage %s is in state %s", stage.getStageId(), state)); } } + + scheduling.set(false); + List nextStagesToSchedule = getStagesReadyForExecution(); + if (!nextStagesToSchedule.isEmpty()) { + startScheduling(nextStagesToSchedule); + } } catch (Throwable t) { queryStateMachine.transitionToFailed(t); throw t; } finally { + scheduling.set(false); RuntimeException closeError = new RuntimeException(); - for (StageScheduler scheduler : stageSchedulers.values()) { + for (SqlStageExecution stage : stages) { try { - scheduler.close(); + stageSchedulers.get(stage.getStageId()).close(); } catch (Throwable t) { queryStateMachine.transitionToFailed(t); @@ -676,11 +816,45 @@ private static ListenableFuture whenAllStages(Collection s return future; } + public static StreamingPlanSection extractStreamingSections(SubPlan subPlan) + { + ImmutableList.Builder materializedExchangeChildren = ImmutableList.builder(); + StreamingSubPlan streamingSection = extractStreamingSection(subPlan, materializedExchangeChildren); + return new StreamingPlanSection( + streamingSection, + materializedExchangeChildren.build().stream() + .map(SqlQueryScheduler::extractStreamingSections) + .collect(toImmutableList())); + } + + private static StreamingSubPlan extractStreamingSection(SubPlan subPlan, ImmutableList.Builder materializedExchangeChildren) + { + ImmutableList.Builder streamingSources = ImmutableList.builder(); + Set streamingFragmentIds = subPlan.getFragment().getRemoteSourceNodes().stream() + .map(RemoteSourceNode::getSourceFragmentIds) + .flatMap(List::stream) + .collect(toImmutableSet()); + for (SubPlan child : subPlan.getChildren()) { + if (streamingFragmentIds.contains(child.getFragment().getId())) { + streamingSources.add(extractStreamingSection(child, materializedExchangeChildren)); + } + else { + materializedExchangeChildren.add(child); + } + } + return new StreamingSubPlan(subPlan.getFragment(), streamingSources.build()); + } + private interface ExchangeLocationsConsumer { void addExchangeLocations(PlanFragmentId fragmentId, Set tasks, boolean noMoreExchangeLocations); } + private static ExchangeLocationsConsumer discardingLocationConsumer() + { + return (fragmentId, tasks, noMoreExchangeLocations) -> {}; + } + private static class StageLinkage { private final PlanFragmentId currentStageFragmentId; @@ -755,4 +929,58 @@ public void processScheduleResults(StageState newState, Set newTasks } } } + + private static class StreamingPlanSection + { + private final StreamingSubPlan plan; + // materialized exchange children + private final List children; + + public StreamingPlanSection(StreamingSubPlan plan, List children) + { + this.plan = requireNonNull(plan, "plan is null"); + this.children = ImmutableList.copyOf(requireNonNull(children, "children is null")); + } + + public StreamingSubPlan getPlan() + { + return plan; + } + + public List getChildren() + { + return children; + } + } + + /** + * StreamingSubPlan is similar to SubPlan but only contains streaming children + */ + private static class StreamingSubPlan + { + private final PlanFragment fragment; + // streaming children + private final List children; + + public StreamingSubPlan(PlanFragment fragment, List children) + { + this.fragment = requireNonNull(fragment, "fragment is null"); + this.children = ImmutableList.copyOf(requireNonNull(children, "children is null")); + } + + public PlanFragment getFragment() + { + return fragment; + } + + public List getChildren() + { + return children; + } + + public StreamingSubPlan withBucketToPartition(Optional bucketToPartition) + { + return new StreamingSubPlan(fragment.withBucketToPartition(bucketToPartition), children); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java index 30e9e45cc33fa..8929d0096e9ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubPlan.java @@ -21,7 +21,6 @@ import javax.annotation.concurrent.Immutable; import java.util.List; -import java.util.Optional; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableMultiset.toImmutableMultiset; @@ -67,11 +66,6 @@ public List getAllFragments() return fragments.build(); } - public SubPlan withBucketToPartition(Optional bucketToPartition) - { - return new SubPlan(fragment.withBucketToPartition(bucketToPartition), children); - } - public void sanityCheck() { Multiset remoteSourceIds = fragment.getRemoteSourceNodes().stream() From 73a7cd5be349d3dae8606c2ae26e9506bb43d52d Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:46 -0400 Subject: [PATCH 07/66] Add exchange materialization tests --- .../facebook/presto/hive/HiveQueryRunner.java | 44 +++++++ ...stributedAggregationsHivePartitioning.java | 43 ------- ...gregationsWithExchangeMaterialization.java | 66 ++++++++++ ...edQueriesWithExchangeMaterialization.java} | 114 ++++++++++++++++-- .../hive/TestHiveIntegrationSmokeTest.java | 80 ++++++++++-- .../tests/AbstractTestQueryFramework.java | 6 + 6 files changed, 293 insertions(+), 60 deletions(-) delete mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java rename presto-hive/src/test/java/com/facebook/presto/hive/{TestHiveDistributedQueriesHivePartitioning.java => TestHiveDistributedQueriesWithExchangeMaterialization.java} (51%) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java index 0c65f7fad1c9f..d2c7334a783b9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.Session; +import com.facebook.presto.execution.QueryManagerConfig.ExchangeMaterializationStrategy; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; @@ -37,6 +38,11 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN; +import static com.facebook.presto.SystemSessionProperties.EXCHANGE_MATERIALIZATION_STRATEGY; +import static com.facebook.presto.SystemSessionProperties.GROUPED_EXECUTION_FOR_AGGREGATION; +import static com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT; +import static com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG; import static com.facebook.presto.spi.security.SelectedRole.Type.ROLE; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; @@ -60,6 +66,7 @@ private HiveQueryRunner() public static final String HIVE_BUCKETED_CATALOG = "hive_bucketed"; public static final String TPCH_SCHEMA = "tpch"; private static final String TPCH_BUCKETED_SCHEMA = "tpch_bucketed"; + private static final String TEMPORARY_TABLE_SCHEMA = "__temporary_tables__"; private static final DateTimeZone TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas"); public static DistributedQueryRunner createQueryRunner(TpchTable... tables) @@ -112,6 +119,7 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta .put("hive.max-partitions-per-scan", "1000") .put("hive.assume-canonical-partition-keys", "true") .put("hive.collect-column-statistics-on-write", "true") + .put("hive.temporary-table-schema", TEMPORARY_TABLE_SCHEMA) .build(); Map hiveBucketedProperties = ImmutableMap.builder() .putAll(hiveProperties) @@ -133,6 +141,10 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta copyTpchTablesBucketed(queryRunner, "tpch", TINY_SCHEMA_NAME, createBucketedSession(Optional.empty()), tables); } + if (!metastore.getDatabase(TEMPORARY_TABLE_SCHEMA).isPresent()) { + metastore.createDatabase(createDatabaseMetastoreObject(TEMPORARY_TABLE_SCHEMA)); + } + return queryRunner; } catch (Exception e) { @@ -141,6 +153,20 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta } } + public static DistributedQueryRunner createMaterializingQueryRunner(Iterable> tables) + throws Exception + { + return createQueryRunner( + tables, + ImmutableMap.of( + "query.partitioning-provider-catalog", "hive", + "query.exchange-materialization-strategy", "ALL", + "query.hash-partition-count", "11", + "colocated-joins-enabled", "true", + "grouped-execution-for-aggregation-enabled", "true"), + Optional.empty()); + } + private static void setupLogging() { Logging logging = Logging.initialize(); @@ -184,6 +210,24 @@ public static Session createBucketedSession(Optional role) .build(); } + public static Session createMaterializeExchangesSession(Optional role) + { + return testSessionBuilder() + .setIdentity(new Identity( + "hive", + Optional.empty(), + role.map(selectedRole -> ImmutableMap.of("hive", selectedRole)) + .orElse(ImmutableMap.of()))) + .setSystemProperty(PARTITIONING_PROVIDER_CATALOG, HIVE_CATALOG) + .setSystemProperty(EXCHANGE_MATERIALIZATION_STRATEGY, ExchangeMaterializationStrategy.ALL.toString()) + .setSystemProperty(HASH_PARTITION_COUNT, "13") + .setSystemProperty(COLOCATED_JOIN, "true") + .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") + .setCatalog(HIVE_CATALOG) + .setSchema(TPCH_SCHEMA) + .build(); + } + public static void copyTpchTablesBucketed( QueryRunner queryRunner, String sourceCatalog, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java deleted file mode 100644 index af85ed9888f94..0000000000000 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsHivePartitioning.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive; - -import com.facebook.presto.tests.AbstractTestAggregations; -import com.google.common.collect.ImmutableMap; - -import java.util.Optional; - -import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; -import static io.airlift.tpch.TpchTable.getTables; - -public class TestHiveDistributedAggregationsHivePartitioning - extends AbstractTestAggregations -{ - public TestHiveDistributedAggregationsHivePartitioning() - { - super(() -> createQueryRunner(getTables(), ImmutableMap.of("query.partitioning-provider-catalog", "hive"), Optional.empty())); - } - - @Override - public void testExtractDistinctAggregationOptimizer() - { - // row type is not supported by the Hive hash code function - } - - @Override - public void testGroupByRow() - { - // row type is not supported by the Hive hash code function - } -} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java new file mode 100644 index 0000000000000..43a3140254a81 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.tests.AbstractTestAggregations; +import org.testng.annotations.Test; + +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializingQueryRunner; +import static com.facebook.presto.hive.TestHiveIntegrationSmokeTest.assertRemoteMaterializedExchangesCount; +import static io.airlift.tpch.TpchTable.getTables; + +public class TestHiveDistributedAggregationsWithExchangeMaterialization + extends AbstractTestAggregations +{ + public TestHiveDistributedAggregationsWithExchangeMaterialization() + { + super(() -> createMaterializingQueryRunner(getTables())); + } + + @Test + public void testMaterializedExchangesEnabled() + { + assertQuery(getSession(), "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", assertRemoteMaterializedExchangesCount(1)); + } + + @Override + public void testExtractDistinctAggregationOptimizer() + { + // row type is not supported by the Hive hash code function + } + + @Override + public void testGroupByRow() + { + // row type is not supported by the Hive hash code function + } + + @Override + public void testApproximateCountDistinctGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testApproximateCountDistinctGroupByWithStandardError() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testAggregationPushedBelowOuterJoin() + { + // Anonymous row type is not supported in Hive + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesHivePartitioning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java similarity index 51% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesHivePartitioning.java rename to presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java index c0bc657810334..2420765c9b5d2 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesHivePartitioning.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java @@ -15,23 +15,27 @@ import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.tests.AbstractTestDistributedQueries; -import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; -import java.util.Optional; - -import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializingQueryRunner; +import static com.facebook.presto.hive.TestHiveIntegrationSmokeTest.assertRemoteMaterializedExchangesCount; import static com.facebook.presto.sql.tree.ExplainType.Type.LOGICAL; import static com.google.common.collect.Iterables.getOnlyElement; import static io.airlift.tpch.TpchTable.getTables; import static org.testng.Assert.assertEquals; -public class TestHiveDistributedQueriesHivePartitioning +public class TestHiveDistributedQueriesWithExchangeMaterialization extends AbstractTestDistributedQueries { - public TestHiveDistributedQueriesHivePartitioning() + public TestHiveDistributedQueriesWithExchangeMaterialization() + { + super(() -> createMaterializingQueryRunner(getTables())); + } + + @Test + public void testMaterializedExchangesEnabled() { - super(() -> createQueryRunner(getTables(), ImmutableMap.of("query.partitioning-provider-catalog", "hive"), Optional.empty())); + assertQuery(getSession(), "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", assertRemoteMaterializedExchangesCount(1)); } @Override @@ -88,6 +92,102 @@ public void testValues() // decimal type is not supported by the Hive hash code function } + @Override + public void testAntiJoinNullHandling() + { + // Unsupported Hive type: unknown + } + + @Override + public void testApproxSetBigintGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testApproxSetVarcharGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testApproxSetDoubleGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testApproxSetGroupByWithOnlyNullsInOneGroup() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testApproxSetGroupByWithNulls() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testMergeHyperLogLogGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testMergeHyperLogLogGroupByWithNulls() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testP4ApproxSetBigintGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testP4ApproxSetVarcharGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testP4ApproxSetDoubleGroupBy() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testP4ApproxSetGroupByWithOnlyNullsInOneGroup() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testP4ApproxSetGroupByWithNulls() + { + // Unsupported Hive type: HyperLogLog + } + + @Override + public void testHaving3() + { + // Anonymous row type is not supported in Hive + } + + @Override + public void testJoinCoercionOnEqualityComparison() + { + // Anonymous row type is not supported in Hive + } + + @Override + public void testCorrelatedScalarSubqueriesWithScalarAggregation() + { + // Anonymous row type is not supported in Hive + } + @Test public void testExplainOfCreateTableAs() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index 9a6812bf00941..904a3161b499d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -75,6 +75,7 @@ import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveQueryRunner.TPCH_SCHEMA; import static com.facebook.presto.hive.HiveQueryRunner.createBucketedSession; +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializeExchangesSession; import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; import static com.facebook.presto.hive.HiveSessionProperties.RCFILE_OPTIMIZED_WRITER_ENABLED; import static com.facebook.presto.hive.HiveSessionProperties.WRITING_STAGING_FILES_ENABLED; @@ -99,6 +100,7 @@ import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_MATERIALIZED; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textLogicalPlan; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; @@ -114,6 +116,7 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.tpch.TpchTable.CUSTOMER; +import static io.airlift.tpch.TpchTable.LINE_ITEM; import static io.airlift.tpch.TpchTable.ORDERS; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; @@ -134,19 +137,30 @@ public class TestHiveIntegrationSmokeTest { private final String catalog; private final Session bucketedSession; + private final Session materializeExchangesSession; private final TypeTranslator typeTranslator; @SuppressWarnings("unused") public TestHiveIntegrationSmokeTest() { - this(() -> createQueryRunner(ORDERS, CUSTOMER), createBucketedSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), HIVE_CATALOG, new HiveTypeTranslator()); + this(() -> createQueryRunner(ORDERS, CUSTOMER, LINE_ITEM), + createBucketedSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), + createMaterializeExchangesSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), + HIVE_CATALOG, + new HiveTypeTranslator()); } - protected TestHiveIntegrationSmokeTest(QueryRunnerSupplier queryRunnerSupplier, Session bucketedSession, String catalog, TypeTranslator typeTranslator) + protected TestHiveIntegrationSmokeTest( + QueryRunnerSupplier queryRunnerSupplier, + Session bucketedSession, + Session materializeExchangesSession, + String catalog, + TypeTranslator typeTranslator) { super(queryRunnerSupplier); this.catalog = requireNonNull(catalog, "catalog is null"); this.bucketedSession = requireNonNull(bucketedSession, "bucketSession is null"); + this.materializeExchangesSession = requireNonNull(materializeExchangesSession, "materializeExchangesSession is null"); this.typeTranslator = requireNonNull(typeTranslator, "typeTranslator is null"); } @@ -2400,6 +2414,12 @@ public void testMismatchedBucketWithBucketPredicate() @Test public void testMismatchedBucketing() + { + testMismatchedBucketing(getSession()); + testMismatchedBucketing(materializeExchangesSession); + } + + public void testMismatchedBucketing(Session session) { try { assertUpdate( @@ -2417,11 +2437,11 @@ public void testMismatchedBucketing() "SELECT orderkey keyN, comment valueN FROM orders", 15000); - Session withMismatchOptimization = Session.builder(getSession()) + Session withMismatchOptimization = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setCatalogSessionProperty(catalog, "optimize_mismatched_bucket_count", "true") .build(); - Session withoutMismatchOptimization = Session.builder(getSession()) + Session withoutMismatchOptimization = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setCatalogSessionProperty(catalog, "optimize_mismatched_bucket_count", "false") .build(); @@ -2470,8 +2490,48 @@ public void testMismatchedBucketing() } } + @Test + public void testMaterializedPartitioning() + { + // Simple smoke tests for materialized partitioning + // Comprehensive testing is done by TestHiveDistributedAggregationsWithExchangeMaterialization, TestHiveDistributedQueriesWithExchangeMaterialization + + // simple aggregation + assertQuery(materializeExchangesSession, "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", assertRemoteMaterializedExchangesCount(1)); + // simple distinct + assertQuery(materializeExchangesSession, "SELECT distinct orderkey FROM lineitem", assertRemoteMaterializedExchangesCount(1)); + // more complex aggregation + assertQuery(materializeExchangesSession, "SELECT custkey, orderstatus, COUNT(DISTINCT orderkey) FROM orders GROUP BY custkey, orderstatus", assertRemoteMaterializedExchangesCount(2)); + + // join + assertQuery(materializeExchangesSession, "SELECT * FROM (lineitem JOIN orders ON lineitem.orderkey = orders.orderkey) x", assertRemoteMaterializedExchangesCount(2)); + } + + public static Consumer assertRemoteMaterializedExchangesCount(int expectedRemoteExchangesCount) + { + return plan -> + { + int actualRemoteExchangesCount = searchFrom(plan.getRoot()) + .where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == REMOTE_MATERIALIZED) + .findAll() + .size(); + if (actualRemoteExchangesCount != expectedRemoteExchangesCount) { + throw new AssertionError(format( + "Expected [%s] materialized exchanges but found [%s] materialized exchanges.", + expectedRemoteExchangesCount, + actualRemoteExchangesCount)); + } + }; + } + @Test public void testGroupedExecution() + { + testGroupedExecution(getSession()); + testGroupedExecution(materializeExchangesSession); + } + + public void testGroupedExecution(Session session) { try { assertUpdate( @@ -2510,40 +2570,40 @@ public void testGroupedExecution() 10); // NOT grouped execution; default - Session notColocated = Session.builder(getSession()) + Session notColocated = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "false") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "false") .build(); // Co-located JOIN with all groups at once, fixed schedule - Session colocatedAllGroupsAtOnce = Session.builder(getSession()) + Session colocatedAllGroupsAtOnce = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "0") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "false") .build(); // Co-located JOIN, 1 group per worker at a time, fixed schedule - Session colocatedOneGroupAtATime = Session.builder(getSession()) + Session colocatedOneGroupAtATime = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "1") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "false") .build(); // Co-located JOIN with all groups at once, dynamic schedule - Session colocatedAllGroupsAtOnceDynamic = Session.builder(getSession()) + Session colocatedAllGroupsAtOnceDynamic = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "0") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "true") .build(); // Co-located JOIN, 1 group per worker at a time, dynamic schedule - Session colocatedOneGroupAtATimeDynamic = Session.builder(getSession()) + Session colocatedOneGroupAtATimeDynamic = Session.builder(session) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(CONCURRENT_LIFESPANS_PER_NODE, "1") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "true") .build(); // Broadcast JOIN, 1 group per worker at a time - Session broadcastOneGroupAtATime = Session.builder(getSession()) + Session broadcastOneGroupAtATime = Session.builder(session) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, BROADCAST.name()) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index cf6e8b3e70d3e..16ca2278a4d55 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -141,6 +141,12 @@ protected void assertQuery(Session session, @Language("SQL") String actual, @Lan QueryAssertions.assertQuery(queryRunner, session, actual, h2QueryRunner, expected, false, false); } + protected void assertQuery(Session session, @Language("SQL") String sql, Consumer planAssertion) + { + checkArgument(queryRunner instanceof DistributedQueryRunner, "pattern assertion is only supported for DistributedQueryRunner"); + QueryAssertions.assertQuery(queryRunner, session, sql, h2QueryRunner, sql, false, false, planAssertion); + } + protected void assertQuery(Session session, @Language("SQL") String actual, @Language("SQL") String expected, Consumer planAssertion) { checkArgument(queryRunner instanceof DistributedQueryRunner, "pattern assertion is only supported for DistributedQueryRunner"); From fa3a61557abd4488e0b0e09232a304cbd2d73970 Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:47 -0400 Subject: [PATCH 08/66] Refactor presto-hive pom Unify duplicate-finder-maven-plugin overrides --- presto-hive/pom.xml | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 29268ed2ef344..987f13f47c9b5 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -330,6 +330,21 @@ + + + + org.basepom.maven + duplicate-finder-maven-plugin + + + parquet.thrift + about.html + + + + + + default @@ -348,16 +363,6 @@ - - org.basepom.maven - duplicate-finder-maven-plugin - - - parquet.thrift - about.html - - - @@ -374,16 +379,6 @@ - - org.basepom.maven - duplicate-finder-maven-plugin - - - parquet.thrift - about.html - - - From c386452d33fefb8814cb4704e5f5f3bae38787ae Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 17 Apr 2019 14:14:48 -0400 Subject: [PATCH 09/66] Run exchange materialization tests in a separate build --- .travis.yml | 1 + presto-hive/pom.xml | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.travis.yml b/.travis.yml index bd0fe871ad8c8..ae457dd985ebb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ env: - TEST_SPECIFIC_MODULES=presto-accumulo - TEST_SPECIFIC_MODULES=presto-cassandra - TEST_SPECIFIC_MODULES=presto-hive + - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-materialized" - TEST_SPECIFIC_MODULES=presto-main - TEST_SPECIFIC_MODULES=presto-mongodb - TEST_OTHER_MODULES=!presto-tests,!presto-raptor,!presto-accumulo,!presto-cassandra,!presto-hive,!presto-kudu,!presto-docs,!presto-server,!presto-server-rpm,!presto-main,!presto-mongodb diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 987f13f47c9b5..27bd548cade91 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -360,12 +360,31 @@ **/TestHiveClientGlueMetastore.java + **/TestHiveDistributedAggregationsWithExchangeMaterialization.java + **/TestHiveDistributedQueriesWithExchangeMaterialization.java + + test-hive-materialized + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestHiveDistributedAggregationsWithExchangeMaterialization.java + **/TestHiveDistributedQueriesWithExchangeMaterialization.java + + + + + + test-hive-glue From 4727843a94746dbb19efc5b31d1cab1f5555d2e3 Mon Sep 17 00:00:00 2001 From: rongrong Date: Wed, 10 Apr 2019 17:05:22 -0700 Subject: [PATCH 10/66] Reduce usage of resolveFunction --- .../facebook/presto/sql/planner/RowExpressionInterpreter.java | 4 ++-- .../sql/planner/iterative/rule/TestPruneWindowColumns.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 72984ea1c89fb..f3122dcf41030 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -633,10 +633,10 @@ private RowExpression createFailureFunction(RuntimeException exception, Type typ requireNonNull(exception, "Exception is null"); String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); - FunctionHandle jsonParse = metadata.getFunctionManager().resolveFunction(session, QualifiedName.of("json_parse"), fromTypes(VARCHAR)); + FunctionHandle jsonParse = metadata.getFunctionManager().lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)); Object json = functionInvoker.invoke(jsonParse, session.toConnectorSession(), utf8Slice(failureInfo)); - FunctionHandle failureFunction = metadata.getFunctionManager().resolveFunction(session, QualifiedName.of("fail"), fromTypes(JSON)); + FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction(QualifiedName.of("fail"), fromTypes(JSON)); FunctionHandle cast = metadata.getFunctionManager().lookupCast(CAST, UNKNOWN.getTypeSignature(), type.getTypeSignature()); return call(CAST.name(), cast, type, call("fail", failureFunction, UNKNOWN, toRowExpression(json, JSON))); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java index 69cc6fc72468b..fb7bf59f240b4 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java @@ -39,7 +39,6 @@ import java.util.Set; import java.util.function.Predicate; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; @@ -58,7 +57,7 @@ public class TestPruneWindowColumns extends BaseRuleTest { - private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().resolveFunction(TEST_SESSION, QualifiedName.of("min"), fromTypes(BIGINT)); + private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("min"), fromTypes(BIGINT)); private static final List inputSymbolNameList = ImmutableList.of("orderKey", "partitionKey", "hash", "startValue1", "startValue2", "endValue1", "endValue2", "input1", "input2", "unused"); From 1aaac63a865647e4f2746f43d688a7cb7f08a35f Mon Sep 17 00:00:00 2001 From: rongrong Date: Wed, 10 Apr 2019 22:29:05 -0700 Subject: [PATCH 11/66] Change lookupFunction to take String instead of QualifiedName as function name lookupFunction is mainly used by tests and optimization rules to quickly locate a known function. It should only work for internal functions. There's no need to use a QualifiedName. Other function namespaces should not have this interface. --- .../benchmark/CountAggregationBenchmark.java | 3 +-- .../DoubleSumAggregationBenchmark.java | 3 +-- .../presto/benchmark/HandTpchQuery1.java | 9 ++++---- .../presto/benchmark/HandTpchQuery6.java | 3 +-- .../benchmark/HashAggregationBenchmark.java | 3 +-- ...patialPartitioningInternalAggregation.java | 3 +-- .../AbstractTestGeoAggregationFunctions.java | 3 +-- .../presto/metadata/FunctionManager.java | 21 +++++++++--------- .../FunctionImplementationDependency.java | 3 +-- .../sql/planner/LiteralInterpreter.java | 3 +-- .../sql/planner/RowExpressionInterpreter.java | 5 ++--- .../planner/StatisticsAggregationPlanner.java | 22 +++++++++---------- ...RewriteSpatialPartitioningAggregation.java | 2 +- .../rule/SimplifyCountOverConstant.java | 2 +- .../TransformCorrelatedInPredicateToJoin.java | 2 +- .../TransformExistsApplyToLateralNode.java | 6 ++--- .../ImplementIntersectAndExceptAsUnion.java | 2 +- .../OptimizeMixedDistinctAggregations.java | 5 ++--- .../ScalarAggregationToJoinRewriter.java | 2 +- ...uantifiedComparisonApplyToLateralJoin.java | 8 +++---- .../SqlToRowExpressionTranslator.java | 3 +-- .../StandardFunctionResolution.java | 13 +++++------ .../presto/cost/TestCostCalculator.java | 2 +- ...kHashAndStreamingAggregationOperators.java | 5 ++--- .../operator/TestAggregationOperator.java | 3 +-- .../operator/TestHashAggregationOperator.java | 5 ++--- .../operator/TestRealAverageAggregation.java | 3 +-- .../TestScanFilterAndProjectOperator.java | 5 ++--- .../TestStreamingAggregationOperator.java | 5 ++--- .../operator/TestTableFinishOperator.java | 3 +-- .../operator/TestTableWriterOperator.java | 3 +-- .../BenchmarkArrayAggregation.java | 3 +-- .../BenchmarkGroupedTypedHistogram.java | 3 +-- .../TestApproximateCountDistinctBoolean.java | 3 +-- .../TestApproximateCountDistinctDouble.java | 3 +-- .../TestApproximateCountDistinctInteger.java | 3 +-- ...TestApproximateCountDistinctIpAddress.java | 3 +-- .../TestApproximateCountDistinctLong.java | 3 +-- ...stApproximateCountDistinctLongDecimal.java | 3 +-- .../TestApproximateCountDistinctSmallint.java | 3 +-- .../TestApproximateCountDistinctTinyint.java | 3 +-- ...TestApproximateCountDistinctVarBinary.java | 3 +-- .../TestApproximatePercentileAggregation.java | 3 +-- .../aggregation/TestArbitraryAggregation.java | 3 +-- .../aggregation/TestArrayAggregation.java | 3 +-- .../aggregation/TestChecksumAggregation.java | 3 +-- .../TestDoubleHistogramAggregation.java | 3 +-- .../operator/aggregation/TestHistogram.java | 3 +-- .../aggregation/TestMapAggAggregation.java | 3 +-- .../aggregation/TestMapUnionAggregation.java | 15 ++++++------- .../TestMultimapAggAggregation.java | 3 +-- ...TestQuantileDigestAggregationFunction.java | 3 +-- .../TestRealHistogramAggregation.java | 3 +-- .../minmaxby/TestMinMaxByAggregation.java | 5 ++--- .../minmaxby/TestMinMaxByNAggregation.java | 5 ++--- .../scalar/BenchmarkArrayDistinct.java | 3 +-- .../operator/scalar/BenchmarkArrayFilter.java | 3 +-- .../BenchmarkArrayHashCodeOperator.java | 3 +-- .../scalar/BenchmarkArrayIntersect.java | 3 +-- .../operator/scalar/BenchmarkArrayJoin.java | 3 +-- .../operator/scalar/BenchmarkArraySort.java | 3 +-- .../scalar/BenchmarkArrayTransform.java | 3 +-- .../operator/scalar/BenchmarkMapConcat.java | 3 +-- .../scalar/BenchmarkTransformKey.java | 3 +-- .../scalar/BenchmarkTransformValue.java | 3 +-- .../scalar/TestPageProcessorCompiler.java | 7 +++--- .../presto/sql/TestExpressionOptimizer.java | 3 +-- .../presto/sql/TestRowExpressionSerde.java | 3 +-- .../TestEffectivePredicateExtractor.java | 2 +- .../TestRowExpressionDomainTranslator.java | 5 ++--- .../presto/sql/planner/TestTypeValidator.java | 12 +++++----- .../rule/TestMergeAdjacentWindows.java | 2 +- .../rule/TestPruneWindowColumns.java | 2 +- ...stSwapAdjacentWindowsBySpecifications.java | 2 +- .../sql/planner/plan/TestWindowNode.java | 2 +- .../relational/TestDeterminismEvaluator.java | 3 +-- .../relational/TestLogicalRowExpressions.java | 5 ++--- presto-ml/pom.xml | 5 ----- .../ml/TestEvaluateClassifierPredictions.java | 3 +-- 79 files changed, 134 insertions(+), 200 deletions(-) diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java index 0cc604eb4f5fe..317ba4afa1d2e 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java @@ -19,7 +19,6 @@ import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; @@ -44,7 +43,7 @@ protected List createOperatorFactories() OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey"); FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager(); InternalAggregationFunction countFunction = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), fromTypes(BIGINT))); + functionManager.lookupFunction("count", fromTypes(BIGINT))); AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(countFunction.bind(ImmutableList.of(0), Optional.empty())), false); return ImmutableList.of(tableScanOperator, aggregationOperator); } diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java index 87837fc44571c..0a1a9171306cf 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java @@ -20,7 +20,6 @@ import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; @@ -45,7 +44,7 @@ protected List createOperatorFactories() OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "orders", "totalprice"); FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); InternalAggregationFunction doubleSum = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE))); + functionManager.lookupFunction("sum", fromTypes(DOUBLE))); AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(doubleSum.bind(ImmutableList.of(0), Optional.empty())), false); return ImmutableList.of(tableScanOperator, aggregationOperator); } diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java index 1be1603a39699..b81e1fa9b55af 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java @@ -27,7 +27,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.util.DateTimeUtils; import com.google.common.collect.ImmutableList; @@ -61,13 +60,13 @@ public HandTpchQuery1(LocalQueryRunner localQueryRunner) FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager(); longAverage = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("avg"), fromTypes(BIGINT))); + functionManager.lookupFunction("avg", fromTypes(BIGINT))); doubleAverage = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("avg"), fromTypes(DOUBLE))); + functionManager.lookupFunction("avg", fromTypes(DOUBLE))); doubleSum = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE))); + functionManager.lookupFunction("sum", fromTypes(DOUBLE))); countFunction = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); } @Override diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java index 25ffcbfa4fb48..235490d848bec 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java @@ -29,7 +29,6 @@ import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.util.DateTimeUtils; import com.google.common.collect.ImmutableList; @@ -57,7 +56,7 @@ public HandTpchQuery6(LocalQueryRunner localQueryRunner) super(localQueryRunner, "hand_tpch_query_6", 10, 100); FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager(); doubleSum = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE))); + functionManager.lookupFunction("sum", fromTypes(DOUBLE))); } @Override diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java index 076c446e3c9e5..7e5ca1806309d 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; @@ -45,7 +44,7 @@ public HashAggregationBenchmark(LocalQueryRunner localQueryRunner) FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager(); doubleSum = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE))); + functionManager.lookupFunction("sum", fromTypes(DOUBLE))); } @Override diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java index 571d119c00c61..b75e74bd84c06 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; import org.testng.annotations.BeforeClass; @@ -98,7 +97,7 @@ private InternalAggregationFunction getFunction() { FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager(); return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("spatial_partitioning"), fromTypes(GEOMETRY, INTEGER))); + functionManager.lookupFunction("spatial_partitioning", fromTypes(GEOMETRY, INTEGER))); } private List makeGeometries() diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java index 34fbd0ce733c3..7dfaaf80c0be1 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java @@ -22,7 +22,6 @@ import com.facebook.presto.plugin.geospatial.GeoPlugin; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slice; import org.testng.annotations.BeforeClass; @@ -52,7 +51,7 @@ public void registerFunctions() functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions())); FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager(); function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of(getFunctionName()), fromTypes(GEOMETRY))); + functionManager.lookupFunction(getFunctionName(), fromTypes(GEOMETRY))); } protected void assertAggregatedGeometries(String testDescription, String expectedWkt, String... wkts) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index 4030a5a919063..36048bf17b27a 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -66,16 +66,6 @@ public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) return staticFunctionNamespace.getFunctionMetadata(functionHandle); } - /** - * Lookup up a function with a fully qualified name and fully bound types. - * - * @throws PrestoException if function could not be found - */ - public FunctionHandle lookupFunction(QualifiedName name, List parameterTypes) - { - return staticFunctionNamespace.lookupFunction(name, parameterTypes); - } - /** * Resolves a function using the SQL path, and implicit type coercions. * @@ -116,6 +106,17 @@ public FunctionHandle resolveOperator(OperatorType operatorType, List parameterTypes) + { + return staticFunctionNamespace.lookupFunction(QualifiedName.of(name), parameterTypes); + } + public FunctionHandle lookupCast(CastType castType, TypeSignature fromType, TypeSignature toType) { return staticFunctionNamespace.lookupCast(castType, fromType, toType); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java b/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java index c1b47c16510ec..a6c8332b163e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/annotations/FunctionImplementationDependency.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.InvocationConvention; import com.facebook.presto.spi.type.TypeSignature; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import java.util.List; @@ -45,7 +44,7 @@ public FunctionImplementationDependency(String name, List argumen @Override protected FunctionHandle getFunctionHandle(BoundVariables boundVariables, FunctionManager functionManager) { - return functionManager.lookupFunction(QualifiedName.of(name), fromTypeSignatures(applyBoundVariables(argumentTypes, boundVariables))); + return functionManager.lookupFunction(name, fromTypeSignatures(applyBoundVariables(argumentTypes, boundVariables))); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java index 2a5054164fd76..ee935a5494da6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java @@ -52,7 +52,6 @@ import com.facebook.presto.sql.tree.Literal; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.sql.tree.NullLiteral; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.StringLiteral; import com.facebook.presto.sql.tree.TimeLiteral; import com.facebook.presto.sql.tree.TimestampLiteral; @@ -238,7 +237,7 @@ protected Object visitGenericLiteral(GenericLiteral node, ConnectorSession sessi } if (JSON.equals(type)) { - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction("json_parse", fromTypes(VARCHAR)); return functionInvoker.invoke(functionHandle, session, ImmutableList.of(utf8Slice(node.getValue()))); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index f3122dcf41030..8f63e672411e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -37,7 +37,6 @@ import com.facebook.presto.sql.InterpretedFunctionInvoker; import com.facebook.presto.sql.planner.Interpreters.LambdaSymbolResolver; import com.facebook.presto.sql.relational.StandardFunctionResolution; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.util.Failures; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -633,10 +632,10 @@ private RowExpression createFailureFunction(RuntimeException exception, Type typ requireNonNull(exception, "Exception is null"); String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); - FunctionHandle jsonParse = metadata.getFunctionManager().lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)); + FunctionHandle jsonParse = metadata.getFunctionManager().lookupFunction("json_parse", fromTypes(VARCHAR)); Object json = functionInvoker.invoke(jsonParse, session.toConnectorSession(), utf8Slice(failureInfo)); - FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction(QualifiedName.of("fail"), fromTypes(JSON)); + FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction("fail", fromTypes(JSON)); FunctionHandle cast = metadata.getFunctionManager().lookupCast(CAST, UNKNOWN.getTypeSignature(), type.getTypeSignature()); return call(CAST.name(), cast, type, call("fail", failureFunction, UNKNOWN, toRowExpression(json, JSON))); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java index dc78bca690d10..cb28d03317ff7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java @@ -80,9 +80,9 @@ public TableStatisticAggregation createStatisticsAggregation(TableStatisticsMeta if (type != ROW_COUNT) { throw new PrestoException(NOT_SUPPORTED, "Table-wide statistic type not supported: " + type); } - QualifiedName count = QualifiedName.of("count"); + String count = "count"; AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation( - new FunctionCall(count, ImmutableList.of()), + new FunctionCall(QualifiedName.of(count), ImmutableList.of()), functionManager.lookupFunction(count, ImmutableList.of()), Optional.empty()); Symbol symbol = symbolAllocator.newSymbol("rowCount", BIGINT); @@ -111,25 +111,25 @@ private ColumnStatisticsAggregation createColumnAggregation(ColumnStatisticType { switch (statisticType) { case MIN_VALUE: - return createAggregation(QualifiedName.of("min"), input.toSymbolReference(), inputType, inputType); + return createAggregation("min", input.toSymbolReference(), inputType, inputType); case MAX_VALUE: - return createAggregation(QualifiedName.of("max"), input.toSymbolReference(), inputType, inputType); + return createAggregation("max", input.toSymbolReference(), inputType, inputType); case NUMBER_OF_DISTINCT_VALUES: - return createAggregation(QualifiedName.of("approx_distinct"), input.toSymbolReference(), inputType, BIGINT); + return createAggregation("approx_distinct", input.toSymbolReference(), inputType, BIGINT); case NUMBER_OF_NON_NULL_VALUES: - return createAggregation(QualifiedName.of("count"), input.toSymbolReference(), inputType, BIGINT); + return createAggregation("count", input.toSymbolReference(), inputType, BIGINT); case NUMBER_OF_TRUE_VALUES: - return createAggregation(QualifiedName.of("count_if"), input.toSymbolReference(), BOOLEAN, BIGINT); + return createAggregation("count_if", input.toSymbolReference(), BOOLEAN, BIGINT); case TOTAL_SIZE_IN_BYTES: - return createAggregation(QualifiedName.of(SumDataSizeForStats.NAME), input.toSymbolReference(), inputType, BIGINT); + return createAggregation(SumDataSizeForStats.NAME, input.toSymbolReference(), inputType, BIGINT); case MAX_VALUE_SIZE_IN_BYTES: - return createAggregation(QualifiedName.of(MaxDataSizeForStats.NAME), input.toSymbolReference(), inputType, BIGINT); + return createAggregation(MaxDataSizeForStats.NAME, input.toSymbolReference(), inputType, BIGINT); default: throw new IllegalArgumentException("Unsupported statistic type: " + statisticType); } } - private ColumnStatisticsAggregation createAggregation(QualifiedName functionName, SymbolReference input, Type inputType, Type outputType) + private ColumnStatisticsAggregation createAggregation(String functionName, SymbolReference input, Type inputType, Type outputType) { FunctionManager functionManager = metadata.getFunctionManager(); FunctionHandle functionHandle = functionManager.lookupFunction(functionName, TypeSignatureProvider.fromTypes(ImmutableList.of(inputType))); @@ -137,7 +137,7 @@ private ColumnStatisticsAggregation createAggregation(QualifiedName functionName verify(resolvedType.equals(inputType), "resolved function input type does not match the input type: %s != %s", resolvedType, inputType); return new ColumnStatisticsAggregation( new AggregationNode.Aggregation( - new FunctionCall(functionName, ImmutableList.of(input)), + new FunctionCall(QualifiedName.of(functionName), ImmutableList.of(input)), functionHandle, Optional.empty()), outputType); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java index 10e49c3d478c6..5b983842dac78 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java @@ -107,7 +107,7 @@ public Result apply(AggregationNode node, Captures captures, Context context) aggregations.put(entry.getKey(), new Aggregation( new FunctionCall(name, ImmutableList.of(envelopeSymbol.toSymbolReference(), partitionCountSymbol.toSymbolReference())), - metadata.getFunctionManager().lookupFunction(QualifiedName.of(NAME), fromTypes(geometryType, INTEGER)), + metadata.getFunctionManager().lookupFunction(NAME, fromTypes(geometryType, INTEGER)), aggregation.getMask())); } else { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java index a83f3cb23b674..8564296c4c372 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java @@ -77,7 +77,7 @@ public Result apply(AggregationNode parent, Captures captures, Context context) changed = true; aggregations.put(symbol, new AggregationNode.Aggregation( new FunctionCall(QualifiedName.of("count"), ImmutableList.of()), - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of()), + functionManager.lookupFunction("count", ImmutableList.of()), aggregation.getMask())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java index 26b015871d684..775e2a8db122d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java @@ -260,7 +260,7 @@ private AggregationNode.Aggregation countWithFilter(Expression condition) return new AggregationNode.Aggregation( countCall, - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of()), + functionManager.lookupFunction("count", ImmutableList.of()), Optional.empty()); /* mask */ } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java index 3257b2c2f807c..84f791db09b3c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java @@ -78,8 +78,8 @@ public class TransformExistsApplyToLateralNode { private static final Pattern PATTERN = applyNode(); - private static final QualifiedName COUNT = QualifiedName.of("count"); - private static final FunctionCall COUNT_CALL = new FunctionCall(COUNT, ImmutableList.of()); + private static final String COUNT = "count"; + private static final FunctionCall COUNT_CALL = new FunctionCall(QualifiedName.of(COUNT), ImmutableList.of()); private final FunctionManager functionManager; @@ -150,7 +150,7 @@ private Optional rewriteToNonDefaultAggregation(ApplyNode applyNode, C private PlanNode rewriteToDefaultAggregation(ApplyNode parent, Context context) { - Symbol count = context.getSymbolAllocator().newSymbol(COUNT.toString(), BIGINT); + Symbol count = context.getSymbolAllocator().newSymbol(COUNT, BIGINT); Symbol exists = getOnlyElement(parent.getSubqueryAssignments().getSymbols()); return new LateralJoinNode( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java index cf32724b52ec6..7981203360f9d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java @@ -250,7 +250,7 @@ private AggregationNode computeCounts(UnionNode sourceNode, List origina QualifiedName name = QualifiedName.of("count"); aggregations.put(output, new Aggregation( new FunctionCall(name, ImmutableList.of(markers.get(i).toSymbolReference())), - functionManager.lookupFunction(name, fromTypes(BIGINT)), + functionManager.lookupFunction(name.getSuffix(), fromTypes(BIGINT)), Optional.empty())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java index 4cebb5f851d35..f97ce74878088 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java @@ -169,10 +169,9 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext createAggregationNode( COUNT, ImmutableList.of(nonNullableAggregationSourceSymbol.toSymbolReference())), functionManager.lookupFunction( - COUNT, + COUNT.getSuffix(), fromTypeSignatures(scalarAggregationSourceTypeSignatures)), entry.getValue().getMask())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java index 3639078862159..11e1a620d4d00 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java @@ -148,19 +148,19 @@ private PlanNode rewriteQuantifiedApplyNode(ApplyNode node, QuantifiedComparison ImmutableMap.of( minValue, new Aggregation( new FunctionCall(MIN, outputColumnReferences), - functionManager.lookupFunction(MIN, outputColumnTypeSignatures), + functionManager.lookupFunction(MIN.getSuffix(), outputColumnTypeSignatures), Optional.empty()), maxValue, new Aggregation( new FunctionCall(MAX, outputColumnReferences), - functionManager.lookupFunction(MAX, outputColumnTypeSignatures), + functionManager.lookupFunction(MAX.getSuffix(), outputColumnTypeSignatures), Optional.empty()), countAllValue, new Aggregation( new FunctionCall(COUNT, emptyList()), - functionManager.lookupFunction(COUNT, emptyList()), + functionManager.lookupFunction(COUNT.getSuffix(), emptyList()), Optional.empty()), countNonNullValue, new Aggregation( new FunctionCall(COUNT, outputColumnReferences), - functionManager.lookupFunction(COUNT, outputColumnTypeSignatures), + functionManager.lookupFunction(COUNT.getSuffix(), outputColumnTypeSignatures), Optional.empty())), globalAggregation(), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java index fc5b40919c4a0..4344fe3d12182 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java @@ -72,7 +72,6 @@ import com.facebook.presto.sql.tree.NotExpression; import com.facebook.presto.sql.tree.NullIfExpression; import com.facebook.presto.sql.tree.NullLiteral; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Row; import com.facebook.presto.sql.tree.SearchedCaseExpression; import com.facebook.presto.sql.tree.SimpleCaseExpression; @@ -301,7 +300,7 @@ protected RowExpression visitGenericLiteral(GenericLiteral node, Void context) if (JSON.equals(type)) { return call( "json_parse", - functionManager.lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)), + functionManager.lookupFunction("json_parse", fromTypes(VARCHAR)), getType(node), constant(utf8Slice(node.getValue()), VARCHAR)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java index 9d4f92b1d32c6..27b4e7f76a02a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ComparisonExpression; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.List; @@ -57,7 +56,7 @@ public StandardFunctionResolution(FunctionManager functionManager) public FunctionHandle notFunction() { - return functionManager.lookupFunction(QualifiedName.of("not"), fromTypes(BOOLEAN)); + return functionManager.lookupFunction("not", fromTypes(BOOLEAN)); } public boolean isNotFunction(FunctionHandle functionHandle) @@ -67,13 +66,13 @@ public boolean isNotFunction(FunctionHandle functionHandle) public FunctionHandle likeVarcharFunction() { - return functionManager.lookupFunction(QualifiedName.of("LIKE"), fromTypes(VARCHAR, LIKE_PATTERN)); + return functionManager.lookupFunction("LIKE", fromTypes(VARCHAR, LIKE_PATTERN)); } public FunctionHandle likeCharFunction(Type valueType) { checkArgument(valueType instanceof CharType, "Expected CHAR value type"); - return functionManager.lookupFunction(QualifiedName.of("LIKE"), fromTypes(valueType, LIKE_PATTERN)); + return functionManager.lookupFunction("LIKE", fromTypes(valueType, LIKE_PATTERN)); } public boolean isLikeFunction(FunctionHandle functionHandle) @@ -83,7 +82,7 @@ public boolean isLikeFunction(FunctionHandle functionHandle) public FunctionHandle likePatternFunction() { - return functionManager.lookupFunction(QualifiedName.of("LIKE_PATTERN"), fromTypes(VARCHAR, VARCHAR)); + return functionManager.lookupFunction("LIKE_PATTERN", fromTypes(VARCHAR, VARCHAR)); } public boolean isCastFunction(FunctionHandle functionHandle) @@ -128,7 +127,7 @@ public boolean isNegateFunction(FunctionHandle functionHandle) public FunctionHandle arrayConstructor(List argumentTypes) { - return functionManager.lookupFunction(QualifiedName.of(ARRAY_CONSTRUCTOR), fromTypes(argumentTypes)); + return functionManager.lookupFunction(ARRAY_CONSTRUCTOR, fromTypes(argumentTypes)); } public FunctionHandle comparisonFunction(ComparisonExpression.Operator operator, Type leftType, Type rightType) @@ -165,6 +164,6 @@ public FunctionHandle comparisonFunction(ComparisonExpression.Operator operator, public FunctionHandle tryFunction(Type returnType) { - return functionManager.lookupFunction(QualifiedName.of("TRY"), fromTypes(returnType)); + return functionManager.lookupFunction("TRY", fromTypes(returnType)); } } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java index 8587f911313dc..7ee8b02af4445 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java @@ -766,7 +766,7 @@ private AggregationNode aggregation(String id, PlanNode source) { AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation( new FunctionCall(QualifiedName.of("count"), ImmutableList.of()), - metadata.getFunctionManager().lookupFunction(QualifiedName.of("count"), ImmutableList.of()), + metadata.getFunctionManager().lookupFunction("count", ImmutableList.of()), Optional.empty()); return new AggregationNode( diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java index 74bee8a69d16c..5de06019876f0 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java @@ -26,7 +26,6 @@ import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; @@ -84,9 +83,9 @@ public class BenchmarkHashAndStreamingAggregationOperators private static final FunctionManager functionManager = metadata.getFunctionManager(); private static final InternalAggregationFunction LONG_SUM = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT))); + functionManager.lookupFunction("sum", fromTypes(BIGINT))); private static final InternalAggregationFunction COUNT = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); @State(Thread) public static class Context diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java index da375107f0f4d..13e9e704560b5 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterMethod; @@ -164,6 +163,6 @@ private void testMemoryTracking(boolean useSystemMemory) private static InternalAggregationFunction getAggregation(String name, Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(name, fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java index fd7ace6234ae2..30717a14b62ab 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java @@ -32,7 +32,6 @@ import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; @@ -100,7 +99,7 @@ public class TestHashAggregationOperator private static final InternalAggregationFunction LONG_AVERAGE = getAggregation("avg", BIGINT); private static final InternalAggregationFunction LONG_SUM = getAggregation("sum", BIGINT); private static final InternalAggregationFunction COUNT = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); private static final int MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024; @@ -739,7 +738,7 @@ private int getHashCapacity(Operator operator) private static InternalAggregationFunction getAggregation(String name, Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(name, fromTypes(arguments))); } private static class DummySpillerFactory diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java index 3393d8fe9683a..0ba0ededf17ae 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestRealAverageAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.StandardTypes; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -44,7 +43,7 @@ public void setUp() { FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); avgFunction = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("avg"), fromTypes(REAL))); + functionManager.lookupFunction("avg", fromTypes(REAL))); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java index e1001a1a0eb23..a2f5f4ae0d704 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java @@ -37,7 +37,6 @@ import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingSplit; import com.facebook.presto.testing.TestingTransactionHandle; @@ -268,7 +267,7 @@ public void testPageYield() ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder projections = ImmutableList.builder(); for (int i = 0; i < totalColumns; i++) { - projections.add(call("generic_long_page_col", functionManager.lookupFunction(QualifiedName.of("generic_long_page_col" + i), fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); + projections.add(call("generic_long_page_col", functionManager.lookupFunction("generic_long_page_col" + i, fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); } Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections.build(), "key"); Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections.build(), MAX_BATCH_SIZE); @@ -333,7 +332,7 @@ public void testRecordCursorYield() List projections = ImmutableList.of(call( "generic_long_record_cursor", - functionManager.lookupFunction(QualifiedName.of("generic_long_record_cursor"), fromTypes(BIGINT)), + functionManager.lookupFunction("generic_long_record_cursor", fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key"); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java index 93dde1f24d068..02fb7dff61064 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java @@ -23,7 +23,6 @@ import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.testing.MaterializedResult; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterMethod; @@ -54,9 +53,9 @@ public class TestStreamingAggregationOperator private static final FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); private static final InternalAggregationFunction LONG_SUM = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT))); + functionManager.lookupFunction("sum", fromTypes(BIGINT))); private static final InternalAggregationFunction COUNT = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of())); + functionManager.lookupFunction("count", ImmutableList.of())); private ExecutorService executor; private ScheduledExecutorService scheduledExecutor; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java index b4b7fdbcf1c9b..ff41a2ea4b6c6 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java @@ -27,7 +27,6 @@ import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slice; @@ -66,7 +65,7 @@ public class TestTableFinishOperator { private static final FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); private static final InternalAggregationFunction LONG_MAX = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("max"), fromTypes(BIGINT))); + functionManager.lookupFunction("max", fromTypes(BIGINT))); private ScheduledExecutorService scheduledExecutor; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java index 4db39e78a7ca6..8f3cc2474fca3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java @@ -37,7 +37,6 @@ import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import org.testng.annotations.AfterClass; @@ -202,7 +201,7 @@ public void testStatisticsAggregation() .addDriverContext(); FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); InternalAggregationFunction longMaxFunction = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("max"), fromTypes(BIGINT))); + functionManager.lookupFunction("max", fromTypes(BIGINT))); TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator( pageSinkManager, new AggregationOperatorFactory( diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java index 8df07726c88ff..c5364d0dbb411 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkArrayAggregation.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -105,7 +104,7 @@ public void setup() } InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of(name), fromTypes(elementType))); + functionManager.lookupFunction(name, fromTypes(elementType))); accumulator = function.bind(ImmutableList.of(0), Optional.empty()).createAccumulator(); block = createChannel(ARRAY_SIZE, elementType); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java index 2a46f554fa1bd..d66872bda282b 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/BenchmarkGroupedTypedHistogram.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; @@ -158,7 +157,7 @@ private static InternalAggregationFunction getInternalAggregationFunctionVarChar FunctionManager functionManager = getMetadata(groupMode).getFunctionManager(); return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(VARCHAR))); + functionManager.lookupFunction(NAME, fromTypes(VARCHAR))); } private static MetadataManager getMetadata(HistogramGroupImplementation groupMode) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java index 185b23ab72a3b..1def0b9d4465e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctBoolean.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.primitives.Booleans; @@ -35,7 +34,7 @@ public class TestApproximateCountDistinctBoolean public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(BOOLEAN, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(BOOLEAN, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java index 9d5d6af4e7084..fc4e59883382e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctDouble.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -28,7 +27,7 @@ public class TestApproximateCountDistinctDouble public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(DOUBLE, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(DOUBLE, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java index 52d522f7fc677..7f7de8c15c7a7 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctInteger.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctInteger public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(INTEGER, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(INTEGER, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java index 09cca15a61684..d75f09ea0c1e1 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slices; import java.util.concurrent.ThreadLocalRandom; @@ -30,7 +29,7 @@ public class TestApproximateCountDistinctIpAddress public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(IPADDRESS, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(IPADDRESS, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java index d2052505aa0b8..27bd78302bd90 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLong.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctLong public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(BIGINT, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(BIGINT, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java index a6767bd765a07..7afa1ab533440 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctLongDecimal.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slices; import java.util.concurrent.ThreadLocalRandom; @@ -33,7 +32,7 @@ public class TestApproximateCountDistinctLongDecimal public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(LONG_DECIMAL, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(LONG_DECIMAL, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java index ae2518c6d38fb..7940c2530f657 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctSmallint.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctSmallint public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(SMALLINT, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(SMALLINT, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java index e07e52692d93f..4369d3a6fe0d9 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctTinyint.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import java.util.concurrent.ThreadLocalRandom; @@ -29,7 +28,7 @@ public class TestApproximateCountDistinctTinyint public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(TINYINT, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(TINYINT, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java index b71bd13c7b17a..4d0def5faa304 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctVarBinary.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import io.airlift.slice.Slices; import java.util.concurrent.ThreadLocalRandom; @@ -30,7 +29,7 @@ public class TestApproximateCountDistinctVarBinary public InternalAggregationFunction getAggregationFunction() { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("approx_distinct"), fromTypes(VARCHAR, DOUBLE))); + functionManager.lookupFunction("approx_distinct", fromTypes(VARCHAR, DOUBLE))); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java index 38ab4bba02d0d..81853f970de43 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestApproximatePercentileAggregation.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.block.RunLengthEncodedBlock; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -444,7 +443,7 @@ public void testDoublePartialStep() private static InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("approx_percentile"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("approx_percentile", fromTypes(arguments))); } private static RunLengthEncodedBlock createRLEBlock(double percentile, int positionCount) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java index 48976549de8d0..4e22c7a4bca93 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArbitraryAggregation.java @@ -17,7 +17,6 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -167,6 +166,6 @@ public void testValidInt() private static InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("arbitrary"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("arbitrary", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java index cd09c43ca8f91..5c6979b507b52 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestArrayAggregation.java @@ -24,7 +24,6 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.SqlDate; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; import org.testng.internal.collections.Ints; @@ -224,6 +223,6 @@ private GroupedAccumulator createGroupedAccumulator(InternalAggregationFunction private InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("array_agg"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("array_agg", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java index eb082ddc16f24..453b58b906679 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestChecksumAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import org.testng.annotations.Test; import static com.facebook.presto.block.BlockAssertions.createArrayBigintBlock; @@ -127,6 +126,6 @@ private static SqlVarbinary expectedChecksum(Type type, Block block) private InternalAggregationFunction getAggregation(Type argument) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("checksum"), fromTypes(argument))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("checksum", fromTypes(argument))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java index 4285c477201b3..dbcbab76a7450 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestDoubleHistogramAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.MapType; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import org.testng.annotations.Test; @@ -46,7 +45,7 @@ public TestDoubleHistogramAggregation() { FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("numeric_histogram"), fromTypes(BIGINT, DOUBLE, DOUBLE))); + functionManager.lookupFunction("numeric_histogram", fromTypes(BIGINT, DOUBLE, DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java index 75864541d7c66..b9732477781ad 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestHistogram.java @@ -30,7 +30,6 @@ import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; @@ -390,7 +389,7 @@ private InternalAggregationFunction getInternalDefaultVarCharAggregationn() private InternalAggregationFunction getAggregation(Type... arguments) { FunctionManager functionManager = getFunctionManager(NEW); - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(NAME, fromTypes(arguments))); } public FunctionManager getFunctionManager() diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java index e48abe2ec85e3..3ca29f7731fbc 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapAggAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -189,6 +188,6 @@ public void testArrayDoubleMap() private InternalAggregationFunction getAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(NAME, fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java index 34d3ef25549ff..d6c9526f3fe6c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMapUnionAggregation.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.MapType; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -46,7 +45,7 @@ public class TestMapUnionAggregation public void testSimpleWithDuplicates() { MapType mapType = mapType(DOUBLE, VARCHAR); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + FunctionHandle functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); InternalAggregationFunction aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -57,7 +56,7 @@ public void testSimpleWithDuplicates() mapBlockOf(DOUBLE, VARCHAR, ImmutableMap.of(43.0, "ccc", 53.0, "ddd", 13.0, "eee")))); mapType = mapType(DOUBLE, BIGINT); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -68,7 +67,7 @@ public void testSimpleWithDuplicates() mapBlockOf(DOUBLE, BIGINT, ImmutableMap.of(1.0, 44L, 2.0, 44L, 4.0, 44L)))); mapType = mapType(BOOLEAN, BIGINT); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -83,7 +82,7 @@ public void testSimpleWithDuplicates() public void testSimpleWithNulls() { MapType mapType = mapType(DOUBLE, VARCHAR); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + FunctionHandle functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); InternalAggregationFunction aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); Map expected = mapOf(23.0, "aaa", 33.0, null, 43.0, "ccc", 53.0, "ddd"); @@ -102,7 +101,7 @@ public void testSimpleWithNulls() public void testStructural() { MapType mapType = mapType(DOUBLE, new ArrayType(VARCHAR)); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + FunctionHandle functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); InternalAggregationFunction aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -135,7 +134,7 @@ public void testStructural() ImmutableList.of("w", "z"))))); mapType = mapType(DOUBLE, mapType(VARCHAR, VARCHAR)); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, @@ -161,7 +160,7 @@ public void testStructural() ImmutableMap.of("e", "f"))))); mapType = mapType(new ArrayType(VARCHAR), DOUBLE); - functionHandle = functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(mapType)); + functionHandle = functionManager.lookupFunction(NAME, fromTypes(mapType)); aggFunc = functionManager.getAggregateFunctionImplementation(functionHandle); assertAggregation( aggFunc, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java index b4bf501f4e264..d750e62a8ae00 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMultimapAggAggregation.java @@ -25,7 +25,6 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; @@ -182,7 +181,7 @@ private static void testMultimapAgg(Type keyType, List expectedKeys, T private static InternalAggregationFunction getInternalAggregationFunction(Type keyType, Type valueType) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of(NAME), fromTypes(keyType, valueType))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(NAME, fromTypes(keyType, valueType))); } private static void testMultimapAgg(InternalAggregationFunction aggFunc, Type keyType, List expectedKeys, Type valueType, List expectedValues) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java index b9e4f52f3a586..81c787ec488d2 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Joiner; import com.google.common.primitives.Floats; import io.airlift.stats.QuantileDigest; @@ -168,7 +167,7 @@ private InternalAggregationFunction getAggregationFunction(Type... type) { FunctionManager functionManager = METADATA.getFunctionManager(); return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("qdigest_agg"), fromTypes(type))); + functionManager.lookupFunction("qdigest_agg", fromTypes(type))); } private void testAggregationBigint(Block inputBlock, Block weightsBlock, double maxError, long... inputs) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java index 9a18187389115..51ba0025248f3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestRealHistogramAggregation.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -50,7 +49,7 @@ public TestRealHistogramAggregation() TypeRegistry typeRegistry = new TypeRegistry(); FunctionManager functionManager = new FunctionManager(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("numeric_histogram"), fromTypes(BIGINT, REAL, DOUBLE))); + functionManager.lookupFunction("numeric_histogram", fromTypes(BIGINT, REAL, DOUBLE))); factory = function.bind(ImmutableList.of(0, 1, 2), Optional.empty()); input = makeInput(10); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java index 978e6cfb04c66..e9c088e16d74f 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByAggregation.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.SqlDecimal; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -578,11 +577,11 @@ public void testMaxUnknownLongArray() private InternalAggregationFunction getMinByAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("min_by"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("min_by", fromTypes(arguments))); } private InternalAggregationFunction getMaxByAggregation(Type... arguments) { - return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction(QualifiedName.of("max_by"), fromTypes(arguments))); + return functionManager.getAggregateFunctionImplementation(functionManager.lookupFunction("max_by", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java index 2f008a1868b1a..939ce8f552e43 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/minmaxby/TestMinMaxByNAggregation.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -297,12 +296,12 @@ public void testOutOfBound() private InternalAggregationFunction getMaxByAggregation(Type... arguments) { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("max_by"), fromTypes(arguments))); + functionManager.lookupFunction("max_by", fromTypes(arguments))); } private InternalAggregationFunction getMinByAggregation(Type... arguments) { return functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("min_by"), fromTypes(arguments))); + functionManager.lookupFunction("min_by", fromTypes(arguments))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java index dd218329b80d9..2fbe911217918 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java @@ -30,7 +30,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; @@ -115,7 +114,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypes(arrayType)); projectionsBuilder.add(new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(i, arrayType)))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java index b6d744629c1b5..f082be7c6c4af 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java @@ -35,7 +35,6 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import org.openjdk.jmh.annotations.Benchmark; @@ -132,7 +131,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); FunctionHandle greaterThan = functionManager.resolveOperator(GREATER_THAN, fromTypes(BIGINT, BIGINT)); projectionsBuilder.add(new CallExpression(name, functionHandle, arrayType, ImmutableList.of( field(0, arrayType), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java index 621fc503b3481..85a45cc88498d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java @@ -34,7 +34,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -142,7 +141,7 @@ public void setup() throw new UnsupportedOperationException(); } ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypes(arrayType)); projectionsBuilder.add(new CallExpression(name, functionHandle, BIGINT, ImmutableList.of(field(0, arrayType)))); blocks[0] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java index 56ca417cf0f79..6bb2a72a00444 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java @@ -27,7 +27,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -124,7 +123,7 @@ public void setup() ArrayType arrayType = new ArrayType(elementType); MetadataManager metadata = createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of(name), fromTypes(arrayType, arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction(name, fromTypes(arrayType, arrayType)); ImmutableList projections = ImmutableList.of( new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(0, arrayType), field(1, arrayType)))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java index 7072dd202a55a..9d8d63fc28f7d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java @@ -26,7 +26,6 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -93,7 +92,7 @@ public void setup() { MetadataManager metadata = createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of("array_join"), fromTypes(new ArrayType(BIGINT), VARCHAR)); + FunctionHandle functionHandle = functionManager.lookupFunction("array_join", fromTypes(new ArrayType(BIGINT), VARCHAR)); List projections = ImmutableList.of( new CallExpression("array_join", functionHandle, VARCHAR, ImmutableList.of( diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java index d704645a01bdd..43567151df376 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java @@ -28,7 +28,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; @@ -113,7 +112,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of(name), fromTypes(arrayType)); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(name, fromTypes(arrayType)); projectionsBuilder.add(new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(i, arrayType)))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java index 381f183bf66b4..dd4d013aa30eb 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java @@ -32,7 +32,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import org.openjdk.jmh.annotations.Benchmark; @@ -116,7 +115,7 @@ public void setup() for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of("transform"), fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); + FunctionHandle functionHandle = functionManager.lookupFunction("transform", fromTypeSignatures(arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)"))); FunctionHandle greaterThan = functionManager.resolveOperator(GREATER_THAN, fromTypes(BIGINT, BIGINT)); projectionsBuilder.add(new CallExpression("transform", functionHandle, returnType, ImmutableList.of( new InputReferenceExpression(0, arrayType), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java index 5b96a58a758be..e839ec1e85531 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java @@ -26,7 +26,6 @@ import com.facebook.presto.spi.type.MapType; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import org.openjdk.jmh.annotations.Benchmark; @@ -139,7 +138,7 @@ public void setup() ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of(name), fromTypes(mapType, mapType)); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(name, fromTypes(mapType, mapType)); projectionsBuilder.add(new CallExpression( name, functionHandle, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java index a56d9c657b7fd..9731a58881d76 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -127,7 +126,7 @@ public void setup() } MapType mapType = mapType(elementType, elementType); FunctionHandle functionHandle = functionManager.lookupFunction( - QualifiedName.of(name), + name, fromTypeSignatures( mapType.getTypeSignature(), parseTypeSignature(format("function(%s, %s, %s)", type, type, type)))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java index 29028f3bef456..acf75f4e0b84d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slices; import org.openjdk.jmh.annotations.Benchmark; @@ -135,7 +134,7 @@ public void setup() MapType mapType = mapType(elementType, elementType); MapType returnType = mapType(elementType, BOOLEAN); FunctionHandle functionHandle = functionManager.lookupFunction( - QualifiedName.of(name), + name, fromTypeSignatures( mapType.getTypeSignature(), parseTypeSignature(format("function(%s, %s, boolean)", type, type)))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java index 978cb78620450..bc5bcf41a408e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java @@ -28,7 +28,6 @@ import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.relational.DeterminismEvaluator; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -83,7 +82,7 @@ public void testNoCaching() FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); ArrayType arrayType = new ArrayType(VARCHAR); - FunctionHandle functionHandle = functionManager.lookupFunction(QualifiedName.of("concat"), fromTypes(arrayType, arrayType)); + FunctionHandle functionHandle = functionManager.lookupFunction("concat", fromTypes(arrayType, arrayType)); projectionsBuilder.add(new CallExpression("concat", functionHandle, arrayType, ImmutableList.of(field(0, arrayType), field(1, arrayType)))); ImmutableList projections = projectionsBuilder.build(); @@ -123,7 +122,7 @@ public void testSanityFilterOnDictionary() { FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); CallExpression lengthVarchar = new CallExpression( - "length", functionManager.lookupFunction(QualifiedName.of("length"), fromTypes(VARCHAR)), BIGINT, ImmutableList.of(field(0, VARCHAR))); + "length", functionManager.lookupFunction("length", fromTypes(VARCHAR)), BIGINT, ImmutableList.of(field(0, VARCHAR))); FunctionHandle lessThan = functionManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)); CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(lengthVarchar, constant(10L, BIGINT))); @@ -211,7 +210,7 @@ public void testNonDeterministicProject() FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); FunctionHandle lessThan = functionManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)); CallExpression random = new CallExpression( - "random", functionManager.lookupFunction(QualifiedName.of("random"), fromTypes(BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); + "random", functionManager.lookupFunction("random", fromTypes(BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); InputReferenceExpression col0 = field(0, BIGINT); CallExpression lessThanRandomExpression = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(col0, random)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java index 514dc239f90f7..fd5ef1dd4d9d5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java @@ -26,7 +26,6 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; @@ -102,7 +101,7 @@ public void testIfConstantOptimization() @Test public void testCastWithJsonParseOptimization() { - FunctionHandle jsonParseFunctionHandle = functionManager.lookupFunction(QualifiedName.of("json_parse"), fromTypes(VARCHAR)); + FunctionHandle jsonParseFunctionHandle = functionManager.lookupFunction("json_parse", fromTypes(VARCHAR)); // constant FunctionHandle jsonCastFunctionHandle = functionManager.lookupCast(CAST, JSON.getTypeSignature(), parseTypeSignature("array(integer)")); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java index 362e634585137..6a5df34775a2a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java @@ -39,7 +39,6 @@ import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; @@ -233,7 +232,7 @@ private FunctionHandle operator(OperatorType operatorType, Type... types) private FunctionHandle function(String name, Type... types) { - return metadata.getFunctionManager().lookupFunction(QualifiedName.of(name), fromTypes(types)); + return metadata.getFunctionManager().lookupFunction(name, fromTypes(types)); } private JsonCodec getJsonCodec() diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java index 80bf696a03e70..6378fe851be2b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java @@ -140,7 +140,7 @@ public void setUp() public void testAggregation() { FunctionCall functionCall = new FunctionCall(QualifiedName.of("count"), ImmutableList.of()); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction(QualifiedName.of("count"), ImmutableList.of()); + FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction("count", ImmutableList.of()); PlanNode node = new AggregationNode(newId(), filter(baseTableScan, and( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index 15ed0fdc74ac1..d0ac90767d0a9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -30,7 +30,6 @@ import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator.ExtractionResult; import com.facebook.presto.sql.relational.StandardFunctionResolution; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; @@ -902,7 +901,7 @@ public void testFromNullLiteralPredicate() @Test public void testExpressionConstantFolding() { - FunctionHandle hex = metadata.getFunctionManager().lookupFunction(QualifiedName.of("from_hex"), fromTypes(VARCHAR)); + FunctionHandle hex = metadata.getFunctionManager().lookupFunction("from_hex", fromTypes(VARCHAR)); RowExpression originalExpression = greaterThan(C_VARBINARY, call("from_hex", hex, VARBINARY, stringLiteral("123456"))); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE); @@ -1308,7 +1307,7 @@ private RowExpression unprocessableExpression2(VariableReferenceExpression expre private RowExpression randPredicate(VariableReferenceExpression expression) { - RowExpression random = call("random", metadata.getFunctionManager().lookupFunction(QualifiedName.of("random"), fromTypes()), DOUBLE); + RowExpression random = call("random", metadata.getFunctionManager().lookupFunction("random", fromTypes()), DOUBLE); return greaterThan( expression, call(CastType.CAST.name(), metadata.getFunctionManager().lookupCast(CastType.CAST, DOUBLE.getTypeSignature(), expression.getType().getTypeSignature()), expression.getType(), random)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java index dd92f7c32200d..ec32e38d3aa9c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java @@ -148,7 +148,7 @@ public void testValidUnion() public void testValidWindow() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); - FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)); + FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)); FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())); WindowNode.Frame frame = new WindowNode.Frame( @@ -186,7 +186,7 @@ public void testValidAggregation() baseTableScan, ImmutableMap.of(aggregationSymbol, new Aggregation( new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())), - FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)), + FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)), Optional.empty())), singleGroupingSet(ImmutableList.of(columnA, columnB)), ImmutableList.of(), @@ -237,7 +237,7 @@ public void testInvalidAggregationFunctionCall() baseTableScan, ImmutableMap.of(aggregationSymbol, new Aggregation( new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnA.toSymbolReference())), - FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)), + FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)), Optional.empty())), singleGroupingSet(ImmutableList.of(columnA, columnB)), ImmutableList.of(), @@ -258,7 +258,7 @@ public void testInvalidAggregationFunctionSignature() baseTableScan, ImmutableMap.of(aggregationSymbol, new Aggregation( new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())), - FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT)), // should be DOUBLE + FUNCTION_MANAGER.lookupFunction("sum", fromTypes(BIGINT)), // should be DOUBLE Optional.empty())), singleGroupingSet(ImmutableList.of(columnA, columnB)), ImmutableList.of(), @@ -273,7 +273,7 @@ public void testInvalidAggregationFunctionSignature() public void testInvalidWindowFunctionCall() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); - FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)); + FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction("sum", fromTypes(DOUBLE)); FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnA.toSymbolReference())); // should be columnC WindowNode.Frame frame = new WindowNode.Frame( @@ -305,7 +305,7 @@ public void testInvalidWindowFunctionCall() public void testInvalidWindowFunctionSignature() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); - FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT)); // should be DOUBLE + FunctionHandle functionHandle = FUNCTION_MANAGER.lookupFunction("sum", fromTypes(BIGINT)); // should be DOUBLE FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())); WindowNode.Frame frame = new WindowNode.Frame( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java index a941a0d1569da..8c9fd78c911fa 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java @@ -58,7 +58,7 @@ public class TestMergeAdjacentWindows Optional.empty(), Optional.empty()); - private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("avg"), fromTypes(DOUBLE)); + private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction("avg", fromTypes(DOUBLE)); private static final String columnAAlias = "ALIAS_A"; private static final ExpectedValueProvider specificationA = specification(ImmutableList.of(columnAAlias), ImmutableList.of(), ImmutableMap.of()); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java index fb7bf59f240b4..8427deb005342 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java @@ -57,7 +57,7 @@ public class TestPruneWindowColumns extends BaseRuleTest { - private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("min"), fromTypes(BIGINT)); + private static final FunctionHandle FUNCTION_HANDLE = createTestMetadataManager().getFunctionManager().lookupFunction("min", fromTypes(BIGINT)); private static final List inputSymbolNameList = ImmutableList.of("orderKey", "partitionKey", "hash", "startValue1", "startValue2", "endValue1", "endValue2", "input1", "input2", "unused"); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java index a895a29dc56a4..e567d6de37f22 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java @@ -56,7 +56,7 @@ public TestSwapAdjacentWindowsBySpecifications() Optional.empty(), Optional.empty()); - functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("avg"), fromTypes(BIGINT)); + functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction("avg", fromTypes(BIGINT)); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java index c9b380e529528..2d6da80928766 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java @@ -91,7 +91,7 @@ public void testSerializationRoundtrip() throws Exception { Symbol windowSymbol = symbolAllocator.newSymbol("sum", BIGINT); - FunctionHandle functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction(QualifiedName.of("sum"), fromTypes(BIGINT)); + FunctionHandle functionHandle = createTestMetadataManager().getFunctionManager().lookupFunction("sum", fromTypes(BIGINT)); FunctionCall functionCall = new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(columnC.toSymbolReference())); WindowNode.Frame frame = new WindowNode.Frame( RANGE, diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java index 28a5b3b9f184a..f181754bdbdee 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDeterminismEvaluator.java @@ -17,7 +17,6 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -42,7 +41,7 @@ public void testDeterminismEvaluator() CallExpression random = new CallExpression( "random", - functionManager.lookupFunction(QualifiedName.of("random"), fromTypes(BIGINT)), + functionManager.lookupFunction("random", fromTypes(BIGINT)), BIGINT, singletonList(constant(10L, BIGINT))); assertFalse(determinismEvaluator.isDeterministic(random)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java index d415a1d4f36ce..05415005b2e4f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeClass; @@ -112,8 +111,8 @@ public void testDeterminism() { RowExpression a = name("a"); RowExpression b = name("b"); - RowExpression nondeterministic = call("random", functionManager.lookupFunction(QualifiedName.of("random"), fromTypes()), DOUBLE); - RowExpression deterministic = call("length", functionManager.lookupFunction(QualifiedName.of("length"), fromTypes(VARCHAR)), INTEGER); + RowExpression nondeterministic = call("random", functionManager.lookupFunction("random", fromTypes()), DOUBLE); + RowExpression deterministic = call("length", functionManager.lookupFunction("length", fromTypes(VARCHAR)), INTEGER); RowExpression expression = and(and(a, or(b, nondeterministic)), deterministic); diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index e0201cd333cef..14699e39611f1 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -21,11 +21,6 @@ presto-array - - com.facebook.presto - presto-parser - - com.facebook.thirdparty libsvm diff --git a/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java b/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java index dec4e14fdf769..4f8b007a901e6 100644 --- a/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java +++ b/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -45,7 +44,7 @@ public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = functionManager.getAggregateFunctionImplementation( - functionManager.lookupFunction(QualifiedName.of("evaluate_classifier_predictions"), fromTypes(BIGINT, BIGINT))); + functionManager.lookupFunction("evaluate_classifier_predictions", fromTypes(BIGINT, BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); From 9cbe203a37b86d7eb340326a54ec87a150f39e7f Mon Sep 17 00:00:00 2001 From: rongrong Date: Sun, 14 Apr 2019 11:38:11 -0700 Subject: [PATCH 12/66] Add FunctionHandleResolver --- .../metadata/FunctionHandleJacksonModule.java | 28 ++++++ .../metadata/FunctionHandleResolver.java | 21 +++++ .../metadata/FunctionNamespaceFactory.java | 21 +++++ .../presto/metadata/HandleJsonModule.java | 1 + .../presto/metadata/HandleResolver.java | 87 +++++++++++++++++++ ...StaticFunctionNamespaceHandleResolver.java | 26 ++++++ 6 files changed, 184 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java new file mode 100644 index 0000000000000..f8a8a90053508 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleJacksonModule.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; + +import javax.inject.Inject; + +public class FunctionHandleJacksonModule + extends AbstractTypedJacksonModule +{ + @Inject + public FunctionHandleJacksonModule(HandleResolver handleResolver) + { + super(FunctionHandle.class, handleResolver::getId, handleResolver::getFunctionHandleClass); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java new file mode 100644 index 0000000000000..b8ed747087447 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; + +public interface FunctionHandleResolver +{ + Class getFunctionHandleClass(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java new file mode 100644 index 0000000000000..05b2dd6f1ba67 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +public interface FunctionNamespaceFactory +{ + String getName(); + + FunctionHandleResolver getHandleResolver(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java index bd70c286b33a9..62238515de1ec 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java @@ -35,6 +35,7 @@ public void configure(Binder binder) jsonBinder(binder).addModuleBinding().to(IndexHandleJacksonModule.class); jsonBinder(binder).addModuleBinding().to(TransactionHandleJacksonModule.class); jsonBinder(binder).addModuleBinding().to(PartitioningHandleJacksonModule.class); + jsonBinder(binder).addModuleBinding().to(FunctionHandleJacksonModule.class); binder.bind(HandleResolver.class).in(Scopes.SINGLETON); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java index 4a45072216238..cea0264385aca 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.split.EmptySplitHandleResolver; import javax.inject.Inject; @@ -45,6 +46,7 @@ public class HandleResolver { private final ConcurrentMap handleResolvers = new ConcurrentHashMap<>(); + private final ConcurrentMap functionHandleResolvers = new ConcurrentHashMap<>(); @Inject public HandleResolver() @@ -53,6 +55,8 @@ public HandleResolver() handleResolvers.put("$system", new MaterializedHandleResolver(new SystemHandleResolver())); handleResolvers.put("$info_schema", new MaterializedHandleResolver(new InformationSchemaHandleResolver())); handleResolvers.put("$empty", new MaterializedHandleResolver(new EmptySplitHandleResolver())); + + functionHandleResolvers.put("$static", new MaterializedFunctionHandleResolver(new StaticFunctionNamespaceHandleResolver())); } public void addConnectorName(String name, ConnectorHandleResolver resolver) @@ -64,6 +68,14 @@ public void addConnectorName(String name, ConnectorHandleResolver resolver) "Connector '%s' is already assigned to resolver: %s", name, existingResolver); } + public void addFunctionNamepsace(String name, FunctionHandleResolver resolver) + { + requireNonNull(name, "name is null"); + requireNonNull(resolver, "resolver is null"); + MaterializedFunctionHandleResolver existingResolver = functionHandleResolvers.putIfAbsent(name, new MaterializedFunctionHandleResolver(resolver)); + checkState(existingResolver == null || existingResolver.equals(resolver), "Name %s is already assigned to function resolver: %s", name, existingResolver); + } + public String getId(ConnectorTableHandle tableHandle) { return getId(tableHandle, MaterializedHandleResolver::getTableHandleClass); @@ -109,6 +121,11 @@ public String getId(ConnectorTransactionHandle transactionHandle) return getId(transactionHandle, MaterializedHandleResolver::getTransactionHandleClass); } + public String getId(FunctionHandle functionHandle) + { + return getFunctionNamespaceId(functionHandle, MaterializedFunctionHandleResolver::getFunctionHandleClass); + } + public Class getTableHandleClass(String id) { return resolverFor(id).getTableHandleClass().orElseThrow(() -> new IllegalArgumentException("No resolver for " + id)); @@ -154,6 +171,11 @@ public Class getTransactionHandleClass(Str return resolverFor(id).getTransactionHandleClass().orElseThrow(() -> new IllegalArgumentException("No resolver for " + id)); } + public Class getFunctionHandleClass(String id) + { + return resolverForFunctionNamespace(id).getFunctionHandleClass().orElseThrow(() -> new IllegalArgumentException("No resolver for " + id)); + } + private MaterializedHandleResolver resolverFor(String id) { MaterializedHandleResolver resolver = handleResolvers.get(id); @@ -161,6 +183,13 @@ private MaterializedHandleResolver resolverFor(String id) return resolver; } + private MaterializedFunctionHandleResolver resolverForFunctionNamespace(String id) + { + MaterializedFunctionHandleResolver resolver = functionHandleResolvers.get(id); + checkArgument(resolver != null, "No handle resolver for function namespace: %s", id); + return resolver; + } + private String getId(T handle, Function>> getter) { for (Entry entry : handleResolvers.entrySet()) { @@ -175,6 +204,20 @@ private String getId(T handle, Function String getFunctionNamespaceId(T handle, Function>> getter) + { + for (Entry entry : functionHandleResolvers.entrySet()) { + try { + if (getter.apply(entry.getValue()).map(clazz -> clazz.isInstance(handle)).orElse(false)) { + return entry.getKey(); + } + } + catch (UnsupportedOperationException ignored) { + } + } + throw new IllegalArgumentException("No function namespace for handle: " + handle); + } + private static class MaterializedHandleResolver { private final Optional> tableHandle; @@ -282,4 +325,48 @@ public int hashCode() return Objects.hash(tableHandle, layoutHandle, columnHandle, split, indexHandle, outputTableHandle, insertTableHandle, partitioningHandle, transactionHandle); } } + + private static class MaterializedFunctionHandleResolver + { + private final Optional> functionHandle; + + public MaterializedFunctionHandleResolver(FunctionHandleResolver resolver) + { + functionHandle = getHandleClass(resolver::getFunctionHandleClass); + } + + private static Optional> getHandleClass(Supplier> callable) + { + try { + return Optional.of(callable.get()); + } + catch (UnsupportedOperationException e) { + return Optional.empty(); + } + } + + public Optional> getFunctionHandleClass() + { + return functionHandle; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MaterializedFunctionHandleResolver that = (MaterializedFunctionHandleResolver) o; + return Objects.equals(functionHandle, that.functionHandle); + } + + @Override + public int hashCode() + { + return Objects.hash(functionHandle); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java new file mode 100644 index 0000000000000..7c5ac338070c1 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; + +public class StaticFunctionNamespaceHandleResolver + implements FunctionHandleResolver +{ + @Override + public Class getFunctionHandleClass() + { + return null; + } +} From 8634f9cf796709dc5dc4fc5ae8d907ce5285d33f Mon Sep 17 00:00:00 2001 From: rongrong Date: Wed, 10 Apr 2019 16:24:17 -0700 Subject: [PATCH 13/66] Add abstraction to FunctionHandle --- .../presto/cost/ScalarStatsCalculator.java | 3 +- .../presto/metadata/FunctionManager.java | 10 +-- .../presto/metadata/FunctionNamespace.java | 31 +++++++ .../metadata/StaticFunctionNamespace.java | 31 ++++--- .../StaticFunctionNamespaceFactory.java | 30 +++++++ ...StaticFunctionNamespaceHandleResolver.java | 2 +- .../StandardFunctionResolution.java | 11 ++- .../metadata/TestStaticFunctionNamespace.java | 8 +- .../presto/sql/TestRowExpressionSerde.java | 2 + .../assertions/RowExpressionVerifier.java | 4 +- .../sql/planner/plan/TestWindowNode.java | 54 ++++++++---- .../presto/metadata/StaticFunctionHandle.java | 82 +++++++++++++++++++ .../presto/spi/function/FunctionHandle.java | 57 +------------ 13 files changed, 230 insertions(+), 95 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 5abe65d9c9429..28d5ae2826eb5 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -68,6 +68,7 @@ import static java.lang.Double.isFinite; import static java.lang.Double.isNaN; import static java.lang.Math.abs; +import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; @@ -239,7 +240,7 @@ private SymbolStatsEstimate computeNegationStatistics(CallExpression call, Void .setHighValue(-stats.getLowValue()) .build(); } - throw new IllegalStateException("Unexpected sign: " + call.getFunctionHandle().getSignature()); + throw new IllegalStateException(format("Unexpected sign: %s(%s)" + call.getDisplayName(), call.getFunctionHandle())); } private SymbolStatsEstimate computeArithmeticBinaryStatistics(CallExpression call, Void context) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index 36048bf17b27a..c7fd8985d5b3b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -61,11 +61,6 @@ public List listFunctions() return staticFunctionNamespace.listFunctions(); } - public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) - { - return staticFunctionNamespace.getFunctionMetadata(functionHandle); - } - /** * Resolves a function using the SQL path, and implicit type coercions. * @@ -81,6 +76,11 @@ public FunctionHandle resolveFunction(Session session, QualifiedName name, List< return staticFunctionNamespace.resolveFunction(name, parameterTypes); } + public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + { + return staticFunctionNamespace.getFunctionMetadata(functionHandle); + } + public WindowFunctionSupplier getWindowFunctionImplementation(FunctionHandle functionHandle) { return staticFunctionNamespace.getWindowFunctionImplementation(functionHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java new file mode 100644 index 0000000000000..1921583429f57 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.sql.analyzer.TypeSignatureProvider; +import com.facebook.presto.sql.tree.QualifiedName; + +import java.util.List; + +public interface FunctionNamespace +{ + void addFunctions(List functions); + + List listFunctions(); + + FunctionHandle resolveFunction(QualifiedName name, List parameterTypes); + + FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java index 334f611ab24bc..4d9933a5d67c2 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java @@ -358,6 +358,7 @@ @ThreadSafe class StaticFunctionNamespace + implements FunctionNamespace { private final TypeManager typeManager; private final LoadingCache specializedFunctionKeyCache; @@ -664,6 +665,7 @@ public StaticFunctionNamespace( addFunctions(builder.getFunctions()); } + @Override public final synchronized void addFunctions(List functions) { for (SqlFunction function : functions) { @@ -674,6 +676,7 @@ public final synchronized void addFunctions(List function this.functions = new FunctionMap(this.functions, functions); } + @Override public List listFunctions() { return functions.list().stream() @@ -686,18 +689,20 @@ public boolean isAggregationFunction(QualifiedName name) return Iterables.any(functions.get(name), function -> function.getSignature().getKind() == AGGREGATE); } + @Override public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) { + checkArgument(functionHandle instanceof StaticFunctionHandle, "Expect StaticFunctionHandle"); + Signature signature = ((StaticFunctionHandle) functionHandle).getSignature(); SpecializedFunctionKey functionKey; try { - functionKey = specializedFunctionKeyCache.getUnchecked(functionHandle.getSignature()); + functionKey = specializedFunctionKeyCache.getUnchecked(signature); } catch (UncheckedExecutionException e) { throwIfInstanceOf(e.getCause(), PrestoException.class); throw e; } SqlFunction function = functionKey.getFunction(); - Signature signature = functionHandle.getSignature(); Optional operatorType = tryGetOperatorType(signature.getName()); if (operatorType.isPresent()) { return new FunctionMetadata( @@ -728,7 +733,7 @@ public FunctionHandle lookupFunction(QualifiedName name, List match = matchFunctionExact(exactCandidates, parameterTypes); if (match.isPresent()) { - return new FunctionHandle(match.get()); + return new StaticFunctionHandle(match.get()); } List genericCandidates = allCandidates.stream() @@ -737,12 +742,13 @@ public FunctionHandle lookupFunction(QualifiedName name, List parameterTypes) { try { @@ -757,7 +763,7 @@ public FunctionHandle resolveFunction(QualifiedName name, List allCandidates = functions.get(name); Optional match = matchFunctionWithCoercion(allCandidates, parameterTypes); if (match.isPresent()) { - return new FunctionHandle(match.get()); + return new StaticFunctionHandle(match.get()); } if (name.getSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { @@ -770,7 +776,7 @@ public FunctionHandle resolveFunction(QualifiedName name, List> toTypes(List typeSignatureProviders, TypeManager typeManager) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java new file mode 100644 index 0000000000000..14e4fed325e99 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +public class StaticFunctionNamespaceFactory + implements FunctionNamespaceFactory +{ + @Override + public String getName() + { + return "$static"; + } + + @Override + public FunctionHandleResolver getHandleResolver() + { + return new StaticFunctionNamespaceHandleResolver(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java index 7c5ac338070c1..19792a7eccaa1 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java @@ -21,6 +21,6 @@ public class StaticFunctionNamespaceHandleResolver @Override public Class getFunctionHandleClass() { - return null; + return StaticFunctionHandle.class; } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java index 27b4e7f76a02a..554dacff5ac8f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java @@ -22,9 +22,11 @@ import com.facebook.presto.sql.tree.ComparisonExpression; import java.util.List; +import java.util.Optional; import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.BETWEEN; import static com.facebook.presto.spi.function.OperatorType.DIVIDE; import static com.facebook.presto.spi.function.OperatorType.EQUAL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; @@ -92,7 +94,7 @@ public boolean isCastFunction(FunctionHandle functionHandle) public boolean isBetweenFunction(FunctionHandle functionHandle) { - return functionHandle.getSignature().getName().equals(mangleOperatorName(OperatorType.BETWEEN.name())); + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(BETWEEN)); } public FunctionHandle arithmeticFunction(ArithmeticBinaryExpression.Operator operator, Type leftType, Type rightType) @@ -122,7 +124,7 @@ public FunctionHandle arithmeticFunction(ArithmeticBinaryExpression.Operator ope public boolean isNegateFunction(FunctionHandle functionHandle) { - return functionHandle.getSignature().getName().equals(mangleOperatorName(NEGATION.name())); + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(NEGATION)); } public FunctionHandle arrayConstructor(List argumentTypes) @@ -166,4 +168,9 @@ public FunctionHandle tryFunction(Type returnType) { return functionManager.lookupFunction("TRY", fromTypes(returnType)); } + + public boolean isTryFunction(FunctionHandle functionHandle) + { + return functionManager.getFunctionMetadata(functionHandle).getName().equals("TRY"); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java index 468610164df0f..c71cdd96a9972 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java @@ -71,7 +71,7 @@ public void testIdentityCast() TypeRegistry typeManager = new TypeRegistry(); StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); FunctionHandle exactOperator = staticFunctionNamespace.lookupCast(CastType.CAST, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()); - assertEquals(exactOperator, new FunctionHandle(new Signature(mangleOperatorName(CAST.name()), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); + assertEquals(exactOperator, new StaticFunctionHandle(new Signature(mangleOperatorName(CAST.name()), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); } @Test @@ -91,7 +91,7 @@ public void testExactMatchBeforeCoercion() if (function.getSignature().getArgumentTypes().stream().anyMatch(TypeSignature::isCalculated)) { continue; } - FunctionHandle exactOperator = staticFunctionNamespace.resolveOperator(operatorType, fromTypeSignatures(function.getSignature().getArgumentTypes())); + StaticFunctionHandle exactOperator = (StaticFunctionHandle) staticFunctionNamespace.resolveOperator(operatorType, fromTypeSignatures(function.getSignature().getArgumentTypes())); assertEquals(exactOperator.getSignature(), function.getSignature()); foundOperator = true; } @@ -108,7 +108,7 @@ public void testMagicLiteralFunction() TypeRegistry typeManager = new TypeRegistry(); StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - FunctionHandle functionHandle = staticFunctionNamespace.resolveFunction(QualifiedName.of(signature.getName()), fromTypeSignatures(signature.getArgumentTypes())); + StaticFunctionHandle functionHandle = (StaticFunctionHandle) staticFunctionNamespace.resolveFunction(QualifiedName.of(signature.getName()), fromTypeSignatures(signature.getArgumentTypes())); assertEquals(staticFunctionNamespace.getFunctionMetadata(functionHandle).getArgumentTypes(), ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT))); assertEquals(signature.getReturnType().getBase(), StandardTypes.TIMESTAMP_WITH_TIME_ZONE); } @@ -372,7 +372,7 @@ public ResolveFunctionAssertion forParameters(String... parameters) public ResolveFunctionAssertion returns(SignatureBuilder functionSignature) { - FunctionHandle expectedFunction = new FunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); + FunctionHandle expectedFunction = new StaticFunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); FunctionHandle actualFunction = resolveFunctionHandle(); assertEquals(expectedFunction, actualFunction); return this; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java index 6a5df34775a2a..4bda1a3160a1a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java @@ -16,6 +16,7 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.block.BlockJsonSerde; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.block.Block; @@ -240,6 +241,7 @@ private JsonCodec getJsonCodec() { Module module = binder -> { binder.install(new JsonModule()); + binder.install(new HandleJsonModule()); configBinder(binder).bindConfig(FeaturesConfig.class); binder.bind(TypeManager.class).to(TypeRegistry.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index aff32bf762e2d..9d3353c61a0d6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -110,7 +110,7 @@ protected Boolean visitNode(Node node, RowExpression context) @Override protected Boolean visitTryExpression(TryExpression expected, RowExpression actual) { - if (!(actual instanceof CallExpression) || !((CallExpression) actual).getFunctionHandle().getSignature().getName().equals("TRY")) { + if (!(actual instanceof CallExpression) || !functionResolution.isTryFunction(((CallExpression) actual).getFunctionHandle())) { return false; } @@ -473,7 +473,7 @@ protected Boolean visitFunctionCall(FunctionCall expected, RowExpression actual) } CallExpression actualFunction = (CallExpression) actual; - if (!expected.getName().equals(QualifiedName.of(actualFunction.getFunctionHandle().getSignature().getName()))) { + if (!expected.getName().equals(QualifiedName.of(metadata.getFunctionManager().getFunctionMetadata(actualFunction.getFunctionHandle()).getName()))) { return false; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java index 2d6da80928766..92f3e4052e652 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.server.SliceDeserializer; import com.facebook.presto.server.SliceSerializer; import com.facebook.presto.spi.block.SortOrder; @@ -25,11 +26,15 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.QualifiedName; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonModule; import io.airlift.slice.Slice; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -45,6 +50,8 @@ import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.BoundType.UNBOUNDED_FOLLOWING; import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.BoundType.UNBOUNDED_PRECEDING; import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.WindowType.RANGE; +import static io.airlift.json.JsonBinder.jsonBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static org.testng.Assert.assertEquals; public class TestWindowNode @@ -55,21 +62,12 @@ public class TestWindowNode private Symbol columnB; private Symbol columnC; - private final ObjectMapper objectMapper; + private final JsonCodec codec; public TestWindowNode() + throws Exception { - // dependencies copied from ServerMainModule.java to avoid depending on whole ServerMainModule here - SqlParser sqlParser = new SqlParser(); - ObjectMapperProvider provider = new ObjectMapperProvider(); - provider.setJsonSerializers(ImmutableMap.of( - Slice.class, new SliceSerializer(), - Expression.class, new Serialization.ExpressionSerializer())); - provider.setJsonDeserializers(ImmutableMap.of( - Slice.class, new SliceDeserializer(), - Expression.class, new Serialization.ExpressionDeserializer(sqlParser), - FunctionCall.class, new Serialization.FunctionCallDeserializer(sqlParser))); - objectMapper = provider.get(); + codec = getJsonCodec(); } @BeforeClass @@ -120,9 +118,9 @@ public void testSerializationRoundtrip() prePartitionedInputs, 0); - String json = objectMapper.writeValueAsString(windowNode); + String json = codec.toJson(windowNode); - WindowNode actualNode = objectMapper.readValue(json, WindowNode.class); + WindowNode actualNode = codec.fromJson(json); assertEquals(actualNode.getId(), windowNode.getId()); assertEquals(actualNode.getSpecification(), windowNode.getSpecification()); @@ -137,4 +135,28 @@ private static PlanNodeId newId() { return new PlanNodeId(UUID.randomUUID().toString()); } + + private JsonCodec getJsonCodec() + throws Exception + { + Module module = binder -> { + SqlParser sqlParser = new SqlParser(); + binder.install(new JsonModule()); + binder.install(new HandleJsonModule()); + binder.bind(SqlParser.class).toInstance(sqlParser); + jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); + jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); + jsonBinder(binder).addSerializerBinding(Expression.class).to(Serialization.ExpressionSerializer.class); + jsonBinder(binder).addDeserializerBinding(Expression.class).to(Serialization.ExpressionDeserializer.class); + jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(Serialization.FunctionCallDeserializer.class); + jsonCodecBinder(binder).bindJsonCodec(WindowNode.class); + }; + Bootstrap app = new Bootstrap(ImmutableList.of(module)); + Injector injector = app + .strictConfig() + .doNotInitializeLogging() + .quiet() + .initialize(); + return injector.getInstance(new Key>() {}); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java new file mode 100644 index 0000000000000..ce5e7fa82d3e3 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.CatalogSchemaName; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.Signature; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class StaticFunctionHandle + implements FunctionHandle +{ + private static final CatalogSchemaName STATIC_FUNCTION_NAMESPACE_CATALOG_SCHEMA_NAME = new CatalogSchemaName("static", "system"); + private final Signature signature; + + @JsonCreator + public StaticFunctionHandle(@JsonProperty("signature") Signature signature) + { + this.signature = requireNonNull(signature, "signature is null"); + checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); + } + + @JsonProperty + public Signature getSignature() + { + return signature; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StaticFunctionHandle that = (StaticFunctionHandle) o; + return Objects.equals(signature, that.signature); + } + + @Override + public CatalogSchemaName getCatalogSchemaName() + { + return STATIC_FUNCTION_NAMESPACE_CATALOG_SCHEMA_NAME; + } + + @Override + public int hashCode() + { + return Objects.hash(signature); + } + + @Override + public String toString() + { + return signature.toString(); + } + + private static void checkArgument(boolean condition, String message, Object... args) + { + if (!condition) { + throw new IllegalArgumentException(String.format(message, args)); + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java index 6caf4b1a0aab3..806a90be50ab4 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java @@ -13,63 +13,14 @@ */ package com.facebook.presto.spi.function; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Objects; - -import static java.util.Objects.requireNonNull; +import com.facebook.presto.spi.CatalogSchemaName; /** * @apiNote FunctionHandle is a unique handle to identify the function implementation from namespaces. * However, currently it is still under changes, so please don't assume it is backward compatible. */ -public class FunctionHandle -{ - private final Signature signature; - - @JsonCreator - public FunctionHandle(@JsonProperty("signature") Signature signature) - { - this.signature = requireNonNull(signature, "signature is null"); - checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); - } - - @JsonProperty - public Signature getSignature() - { - return signature; - } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FunctionHandle that = (FunctionHandle) o; - return Objects.equals(signature, that.signature); - } - - @Override - public int hashCode() - { - return Objects.hash(signature); - } - - @Override - public String toString() - { - return signature.toString(); - } - - private static void checkArgument(boolean condition, String message, Object... args) - { - if (!condition) { - throw new IllegalArgumentException(String.format(message, args)); - } - } +public interface FunctionHandle +{ + CatalogSchemaName getCatalogSchemaName(); } From 44a9c7700e77008a8c42027735e32f262b844891 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Mon, 8 Apr 2019 10:16:52 -0700 Subject: [PATCH 14/66] Rename PrestoNode to InternalNode Extracted-From: https://github.com/prestosql/presto --- .../connector/jmx/TestJmxSplitManager.java | 4 ++-- .../connector/system/NodeSystemTable.java | 6 +++--- .../presto/metadata/DiscoveryNodeManager.java | 10 +++++----- .../presto/metadata/InMemoryNodeManager.java | 2 +- .../{PrestoNode.java => InternalNode.java} | 6 +++--- .../presto/testing/TestingNodeManager.java | 4 ++-- .../execution/BenchmarkNodeScheduler.java | 4 ++-- .../presto/execution/TestNodeScheduler.java | 18 +++++++++--------- .../execution/TestSqlStageExecution.java | 4 ++-- .../scheduler/TestFixedCountScheduler.java | 4 ++-- .../TestSourcePartitionedScheduler.java | 16 ++++++++-------- .../memory/LowMemoryKillerTestingUtils.java | 4 ++-- .../metadata/TestDiscoveryNodeManager.java | 18 +++++++++--------- .../server/remotetask/TestHttpRemoteTask.java | 4 ++-- .../raptor/metadata/TestAssignmentLimiter.java | 6 +++--- .../metadata/TestDatabaseShardManager.java | 4 ++-- .../metadata/TestRaptorSplitManager.java | 6 +++--- .../raptor/storage/TestBucketBalancer.java | 4 ++-- .../raptor/storage/TestShardEjector.java | 4 ++-- 19 files changed, 64 insertions(+), 64 deletions(-) rename presto-main/src/main/java/com/facebook/presto/metadata/{PrestoNode.java => InternalNode.java} (94%) diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java index bc7fcce6827cd..74fca2a720f48 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxSplitManager.java @@ -14,7 +14,7 @@ package com.facebook.presto.connector.jmx; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; @@ -217,6 +217,6 @@ private static List getAllSplits(ConnectorSplitSource splitSourc private static Node createTestingNode(String hostname) { - return new PrestoNode(hostname, URI.create(format("http://%s:8080", hostname)), NodeVersion.UNKNOWN, false); + return new InternalNode(hostname, URI.create(format("http://%s:8080", hostname)), NodeVersion.UNKNOWN, false); } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java index deb93783ec7e2..093e446d3ca17 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java @@ -15,7 +15,7 @@ import com.facebook.presto.metadata.AllNodes; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.InMemoryRecordSet; @@ -95,8 +95,8 @@ private void addRows(Builder table, Set nodes, NodeState state) private static String getNodeVersion(Node node) { - if (node instanceof PrestoNode) { - return ((PrestoNode) node).getNodeVersion().toString(); + if (node instanceof InternalNode) { + return ((InternalNode) node).getNodeVersion().toString(); } return ""; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index a2cbe9154c5bc..33534f8930623 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -81,7 +81,7 @@ public final class DiscoveryNodeManager private final ScheduledExecutorService nodeStateUpdateExecutor; private final ExecutorService nodeStateEventExecutor; private final boolean httpsRequired; - private final PrestoNode currentNode; + private final InternalNode currentNode; @GuardedBy("this") private SetMultimap activeNodesByConnectorId; @@ -121,13 +121,13 @@ public DiscoveryNodeManager( refreshNodesInternal(); } - private static PrestoNode findCurrentNode(List allServices, String currentNodeId, NodeVersion expectedNodeVersion, boolean httpsRequired) + private static InternalNode findCurrentNode(List allServices, String currentNodeId, NodeVersion expectedNodeVersion, boolean httpsRequired) { for (ServiceDescriptor service : allServices) { URI uri = getHttpUri(service, httpsRequired); NodeVersion nodeVersion = getNodeVersion(service); if (uri != null && nodeVersion != null) { - PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion, isCoordinator(service)); + InternalNode node = new InternalNode(service.getNodeId(), uri, nodeVersion, isCoordinator(service)); if (node.getNodeIdentifier().equals(currentNodeId)) { checkState( @@ -217,7 +217,7 @@ private synchronized void refreshNodesInternal() NodeVersion nodeVersion = getNodeVersion(service); boolean coordinator = isCoordinator(service); if (uri != null && nodeVersion != null) { - PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion, coordinator); + InternalNode node = new InternalNode(service.getNodeId(), uri, nodeVersion, coordinator); NodeState nodeState = getNodeState(node); switch (nodeState) { @@ -275,7 +275,7 @@ private synchronized void refreshNodesInternal() } } - private NodeState getNodeState(PrestoNode node) + private NodeState getNodeState(InternalNode node) { if (expectedNodeVersion.equals(node.getNodeVersion())) { if (isNodeShuttingDown(node.getNodeIdentifier())) { diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java index 8df5edec44637..fa90fad75b8db 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java @@ -51,7 +51,7 @@ public InMemoryNodeManager() public InMemoryNodeManager(URI localUri) { - localNode = new PrestoNode("local", localUri, NodeVersion.UNKNOWN, false); + localNode = new InternalNode("local", localUri, NodeVersion.UNKNOWN, false); } public void addCurrentNodeConnector(ConnectorId connectorId) diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/PrestoNode.java b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java similarity index 94% rename from presto-main/src/main/java/com/facebook/presto/metadata/PrestoNode.java rename to presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java index 2dfcfb86bf71f..65b6ca5dc14de 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/PrestoNode.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java @@ -27,7 +27,7 @@ /** * A node is a server in a cluster than can process queries. */ -public class PrestoNode +public class InternalNode implements Node { private final String nodeIdentifier; @@ -35,7 +35,7 @@ public class PrestoNode private final NodeVersion nodeVersion; private final boolean coordinator; - public PrestoNode(String nodeIdentifier, URI httpUri, NodeVersion nodeVersion, boolean coordinator) + public InternalNode(String nodeIdentifier, URI httpUri, NodeVersion nodeVersion, boolean coordinator) { nodeIdentifier = emptyToNull(nullToEmpty(nodeIdentifier).trim()); this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null or empty"); @@ -88,7 +88,7 @@ public boolean equals(Object obj) if ((obj == null) || (getClass() != obj.getClass())) { return false; } - PrestoNode o = (PrestoNode) obj; + InternalNode o = (InternalNode) obj; return nodeIdentifier.equals(o.nodeIdentifier); } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java index ca605fd95cf9d..d76239c726078 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingNodeManager.java @@ -14,7 +14,7 @@ package com.facebook.presto.testing; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeManager; import com.google.common.collect.ImmutableSet; @@ -43,7 +43,7 @@ public TestingNodeManager() public TestingNodeManager(String environment) { - this(environment, new PrestoNode("local", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true), ImmutableSet.of()); + this(environment, new InternalNode("local", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true), ImmutableSet.of()); } public TestingNodeManager(Node localNode) diff --git a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java index 605e1e588bcdc..ec3eae93d7c07 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java @@ -23,7 +23,7 @@ import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.execution.scheduler.NodeSelector; import com.facebook.presto.metadata.InMemoryNodeManager; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; @@ -147,7 +147,7 @@ public void setup() ImmutableList.Builder nodeBuilder = ImmutableList.builder(); for (int i = 0; i < NODES; i++) { - nodeBuilder.add(new PrestoNode("node" + i, URI.create("http://" + addressForHost(i).getHostText()), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node" + i, URI.create("http://" + addressForHost(i).getHostText()), NodeVersion.UNKNOWN, false)); } List nodes = nodeBuilder.build(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory( diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index a4e8c153e0fb4..052042f611870 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -23,7 +23,7 @@ import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.execution.scheduler.NodeSelector; import com.facebook.presto.metadata.InMemoryNodeManager; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; @@ -85,9 +85,9 @@ public void setUp() nodeManager = new InMemoryNodeManager(); ImmutableList.Builder nodeBuilder = ImmutableList.builder(); - nodeBuilder.add(new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); ImmutableList nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() @@ -133,9 +133,9 @@ public void testTopologyAwareScheduling() InMemoryNodeManager nodeManager = new InMemoryNodeManager(); ImmutableList.Builder nodeBuilder = ImmutableList.builder(); - nodeBuilder.add(new PrestoNode("node1", URI.create("http://host1.rack1:11"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false)); - nodeBuilder.add(new PrestoNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node1", URI.create("http://host1.rack1:11"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false)); + nodeBuilder.add(new InternalNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false)); ImmutableList nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); @@ -292,7 +292,7 @@ public void testMaxSplitsPerNode() { TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); - Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + Node newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); @@ -330,7 +330,7 @@ public void testMaxSplitsPerNodePerTask() { TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); - Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + Node newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java index 4e7e1a2902c48..355e45d8681f8 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java @@ -18,7 +18,7 @@ import com.facebook.presto.execution.TestSqlTaskManager.MockLocationFactory; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; import com.facebook.presto.failureDetector.NoOpFailureDetector; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; @@ -126,7 +126,7 @@ private void testFinalStageInfoInternal() if (Thread.interrupted()) { return; } - Node node = new PrestoNode( + Node node = new InternalNode( "source" + i, URI.create("http://10.0.0." + (i / 10_000) + ":" + (i % 10_000)), NodeVersion.UNKNOWN, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java index 03a38e08067e5..2c5e96c1d79c6 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java @@ -18,7 +18,7 @@ import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.TaskId; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; @@ -94,7 +94,7 @@ public void testMultipleNodes() private static List generateRandomNodes(int count) { return IntStream.range(0, count) - .mapToObj(i -> new PrestoNode("other " + i, URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)) + .mapToObj(i -> new InternalNode("other " + i, URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)) .collect(toImmutableList()); } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index c0f502a0a792b..635784b11707e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -28,7 +28,7 @@ import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.spi.ConnectorSplit; @@ -108,9 +108,9 @@ public class TestSourcePartitionedScheduler public TestSourcePartitionedScheduler() { nodeManager.addNode(CONNECTOR_ID, - new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), - new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), - new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); + new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), + new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), + new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); } @BeforeClass @@ -335,9 +335,9 @@ public void testBalancedSplitAssignment() // use private node manager so we can add a node later InMemoryNodeManager nodeManager = new InMemoryNodeManager(); nodeManager.addNode(CONNECTOR_ID, - new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), - new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), - new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); + new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), + new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), + new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); // Schedule 15 splits - there are 3 nodes, each node should get 5 splits @@ -355,7 +355,7 @@ public void testBalancedSplitAssignment() } // Add new node - Node additionalNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + Node additionalNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, additionalNode); // Schedule 5 splits in another query. Since the new node does not have any splits, all 5 splits are assigned to the new node diff --git a/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java b/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java index b29cceaf63c6e..eb31eda96ad1b 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java @@ -15,7 +15,7 @@ package com.facebook.presto.memory; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.memory.MemoryPoolId; @@ -46,7 +46,7 @@ static List toNodeMemoryInfoList(long maxReservedPoolBytes, long max Map reservationByNode = entry.getValue(); for (Map.Entry nodeEntry : reservationByNode.entrySet()) { - PrestoNode node = new PrestoNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false); + InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false); long bytes = nodeEntry.getValue(); if (bytes == 0) { continue; diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java index 6c023425ef9a9..e4f43687a0280 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java @@ -58,8 +58,8 @@ public class TestDiscoveryNodeManager private NodeVersion expectedVersion; private Set activeNodes; private Set inactiveNodes; - private PrestoNode coordinator; - private PrestoNode currentNode; + private InternalNode coordinator; + private InternalNode currentNode; private final PrestoNodeServiceSelector selector = new PrestoNodeServiceSelector(); private HttpClient testHttpClient; @@ -69,17 +69,17 @@ public void setup() testHttpClient = new TestingHttpClient(input -> new TestingResponse(OK, ArrayListMultimap.create(), ACTIVE.name().getBytes())); expectedVersion = new NodeVersion("1"); - coordinator = new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.2.8"), expectedVersion, true); - currentNode = new PrestoNode(nodeInfo.getNodeId(), URI.create("http://192.0.1.1"), expectedVersion, false); + coordinator = new InternalNode(UUID.randomUUID().toString(), URI.create("https://192.0.2.8"), expectedVersion, true); + currentNode = new InternalNode(nodeInfo.getNodeId(), URI.create("http://192.0.1.1"), expectedVersion, false); activeNodes = ImmutableSet.of( currentNode, - new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.1:8080"), expectedVersion, false), - new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.3"), expectedVersion, false), + new InternalNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.1:8080"), expectedVersion, false), + new InternalNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.3"), expectedVersion, false), coordinator); inactiveNodes = ImmutableSet.of( - new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.3.9"), NodeVersion.UNKNOWN, false), - new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.4.9"), new NodeVersion("2"), false)); + new InternalNode(UUID.randomUUID().toString(), URI.create("https://192.0.3.9"), NodeVersion.UNKNOWN, false), + new InternalNode(UUID.randomUUID().toString(), URI.create("https://192.0.4.9"), new NodeVersion("2"), false)); selector.announceNodes(activeNodes, inactiveNodes); } @@ -195,7 +195,7 @@ private synchronized void announceNodes(Set activeNodes, Set inactiv descriptors.add(serviceDescriptor("presto") .setNodeId(node.getNodeIdentifier()) .addProperty("http", node.getHttpUri().toString()) - .addProperty("node_version", ((PrestoNode) node).getNodeVersion().toString()) + .addProperty("node_version", ((InternalNode) node).getNodeVersion().toString()) .addProperty("coordinator", String.valueOf(node.isCoordinator())) .build()); } diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java index ded91e9987bda..b697c77e17d62 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java @@ -30,7 +30,7 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.HandleResolver; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.TaskUpdateRequest; @@ -211,7 +211,7 @@ private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) return httpRemoteTaskFactory.createRemoteTask( TEST_SESSION, new TaskId("test", 1, 2), - new PrestoNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), + new InternalNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), OptionalInt.empty(), diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java index 6c2c0a52d48a0..76453d31bac7e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.metadata; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; @@ -97,8 +97,8 @@ public void testNotEnoughNodes() TestingTicker ticker = new TestingTicker(); HashSet nodes = new HashSet<>(); - Node node1 = new PrestoNode("node1", new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); - Node node2 = new PrestoNode("node2", new URI("http://127.0.0.2/"), NodeVersion.UNKNOWN, false); + Node node1 = new InternalNode("node1", new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); + Node node2 = new InternalNode("node2", new URI("http://127.0.0.2/"), NodeVersion.UNKNOWN, false); nodes.add(node1); nodes.add(node2); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java index d67fa5e43cfed..13a0a21702463 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.metadata; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.util.DaoSupplier; @@ -821,7 +821,7 @@ private static Set nodeIds(Collection nodes) private static Node createTestingNode() { - return new PrestoNode(UUID.randomUUID().toString(), URI.create("http://test"), NodeVersion.UNKNOWN, false); + return new InternalNode(UUID.randomUUID().toString(), URI.create("http://test"), NodeVersion.UNKNOWN, false); } private int columnCount(long tableId) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java index d084a1800ec85..e47c53492bf28 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java @@ -14,8 +14,8 @@ package com.facebook.presto.raptor.metadata; import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.MetadataUtil.TableMetadataBuilder; -import com.facebook.presto.metadata.PrestoNode; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.RaptorConnectorId; @@ -105,7 +105,7 @@ public void setup() NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; String nodeName = UUID.randomUUID().toString(); - nodeManager.addNode(new PrestoNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); + nodeManager.addNode(new InternalNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); RaptorConnectorId connectorId = new RaptorConnectorId("raptor"); metadata = new RaptorMetadata(connectorId.toString(), dbi, shardManager); @@ -174,7 +174,7 @@ public void testAssignRandomNodeWhenBackupAvailable() TestingNodeManager nodeManager = new TestingNodeManager(); RaptorConnectorId connectorId = new RaptorConnectorId("raptor"); NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; - PrestoNode node = new PrestoNode(UUID.randomUUID().toString(), new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); + InternalNode node = new InternalNode(UUID.randomUUID().toString(), new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false); nodeManager.addNode(node); RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, true); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java index 416ce8753e1ce..f7e4281f79a4e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.storage; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.metadata.BucketNode; import com.facebook.presto.raptor.metadata.ColumnInfo; @@ -306,6 +306,6 @@ private static BucketNode bucketNode(int bucketNumber, String nodeIdentifier) private static Node createTestingNode(String nodeIdentifier) { - return new PrestoNode(nodeIdentifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); + return new InternalNode(nodeIdentifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java index 5e05cf242b732..2d34aa7d13197 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java @@ -14,7 +14,7 @@ package com.facebook.presto.raptor.storage; import com.facebook.presto.client.NodeVersion; -import com.facebook.presto.metadata.PrestoNode; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.backup.BackupStore; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.MetadataDao; @@ -199,7 +199,7 @@ private static NodeManager createNodeManager(String current, String... others) private static Node createTestingNode(String identifier) { - return new PrestoNode(identifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); + return new InternalNode(identifier, URI.create("http://test"), NodeVersion.UNKNOWN, false); } private static class TestingBackupStore From 869decbc286070bb308de01b8239851c26113e47 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Mon, 1 Apr 2019 20:53:52 -0700 Subject: [PATCH 15/66] Replace all usages of spi Node with InternalNode Extracted-From: https://github.com/prestosql/presto --- .../connector/ConnectorAwareNodeManager.java | 2 +- .../connector/system/NodeSystemTable.java | 16 +++---- .../connector/system/SystemSplitManager.java | 8 ++-- .../presto/cost/TaskCountEstimator.java | 4 +- .../presto/execution/LocationFactory.java | 6 +-- .../MemoryTrackingRemoteTaskFactory.java | 4 +- .../presto/execution/NodeTaskMap.java | 14 +++--- .../presto/execution/RemoteTaskFactory.java | 4 +- .../presto/execution/SqlStageExecution.java | 12 ++--- .../execution/scheduler/BucketNodeMap.java | 8 ++-- .../DynamicSplitPlacementPolicy.java | 4 +- .../scheduler/FixedBucketNodeMap.java | 10 ++--- .../scheduler/FixedCountScheduler.java | 10 ++--- .../FixedSourcePartitionedScheduler.java | 14 +++--- .../scheduler/NodeAssignmentStats.java | 14 +++--- .../presto/execution/scheduler/NodeMap.java | 20 ++++----- .../execution/scheduler/NodeScheduler.java | 36 +++++++-------- .../execution/scheduler/NodeSelector.java | 10 ++--- .../scheduler/ScaledWriterScheduler.java | 8 ++-- .../scheduler/SimpleNodeSelector.java | 22 ++++----- .../scheduler/SourcePartitionedScheduler.java | 16 +++---- .../scheduler/SplitPlacementPolicy.java | 4 +- .../scheduler/SplitPlacementResult.java | 8 ++-- .../scheduler/SqlQueryScheduler.java | 6 +-- .../scheduler/TopologyAwareNodeSelector.java | 26 +++++------ .../scheduler/group/DynamicBucketNodeMap.java | 8 ++-- .../group/DynamicLifespanScheduler.java | 8 ++-- .../group/FixedLifespanScheduler.java | 14 +++--- .../presto/memory/ClusterMemoryManager.java | 10 ++--- .../presto/memory/RemoteNodeMemory.java | 8 ++-- .../facebook/presto/metadata/AllNodes.java | 19 ++++---- .../presto/metadata/DiscoveryNodeManager.java | 33 +++++++------- .../presto/metadata/InMemoryNodeManager.java | 19 ++++---- .../presto/metadata/InternalNodeManager.java | 9 ++-- .../presto/server/WorkerResource.java | 6 +-- .../remotetask/HttpLocationFactory.java | 6 +-- .../remotetask/HttpRemoteTaskFactory.java | 4 +- .../server/testing/TestingPrestoServer.java | 4 +- .../presto/sql/planner/NodePartitionMap.java | 14 +++--- .../sql/planner/NodePartitioningManager.java | 23 ++++++---- .../sql/planner/SystemPartitioningHandle.java | 4 +- .../execution/BenchmarkNodeScheduler.java | 15 +++---- .../execution/MockRemoteTaskFactory.java | 6 +-- .../presto/execution/TestNodeScheduler.java | 45 +++++++++---------- .../execution/TestSqlStageExecution.java | 3 +- .../presto/execution/TestSqlTaskManager.java | 6 +-- .../scheduler/TestFixedCountScheduler.java | 3 +- .../TestSourcePartitionedScheduler.java | 5 +-- .../memory/LowMemoryKillerTestingUtils.java | 5 +-- .../metadata/TestDiscoveryNodeManager.java | 23 +++++----- .../presto/tests/DistributedQueryRunner.java | 4 +- .../presto/tests/StandaloneQueryRunner.java | 4 +- 52 files changed, 293 insertions(+), 301 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java index 96dd87ec5c561..c2851207ba228 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorAwareNodeManager.java @@ -48,7 +48,7 @@ public Set getAllNodes() @Override public Set getWorkerNodes() { - return nodeManager.getActiveConnectorNodes(connectorId); + return ImmutableSet.copyOf(nodeManager.getActiveConnectorNodes(connectorId)); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java index 093e446d3ca17..ef90e994cb57e 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java @@ -14,13 +14,12 @@ package com.facebook.presto.connector.system; import com.facebook.presto.metadata.AllNodes; -import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.InternalNode; +import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.InMemoryRecordSet; import com.facebook.presto.spi.InMemoryRecordSet.Builder; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.SchemaTableName; @@ -86,22 +85,19 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect return table.build().cursor(); } - private void addRows(Builder table, Set nodes, NodeState state) + private void addRows(Builder table, Set nodes, NodeState state) { - for (Node node : nodes) { + for (InternalNode node : nodes) { table.addRow(node.getNodeIdentifier(), node.getHttpUri().toString(), getNodeVersion(node), isCoordinator(node), state.toString().toLowerCase(Locale.ENGLISH)); } } - private static String getNodeVersion(Node node) + private static String getNodeVersion(InternalNode node) { - if (node instanceof InternalNode) { - return ((InternalNode) node).getNodeVersion().toString(); - } - return ""; + return node.getNodeVersion().toString(); } - private boolean isCoordinator(Node node) + private boolean isCoordinator(InternalNode node) { return nodeManager.getCoordinators().contains(node); } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java index c6eb09863778b..93656fa2b5b26 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemSplitManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.FixedSplitSource; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.SystemTable.Distribution; @@ -72,15 +72,15 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand } ImmutableList.Builder splits = ImmutableList.builder(); - ImmutableSet.Builder nodes = ImmutableSet.builder(); + ImmutableSet.Builder nodes = ImmutableSet.builder(); if (tableDistributionMode == ALL_COORDINATORS) { nodes.addAll(nodeManager.getCoordinators()); } else if (tableDistributionMode == ALL_NODES) { nodes.addAll(nodeManager.getNodes(ACTIVE)); } - Set nodeSet = nodes.build(); - for (Node node : nodeSet) { + Set nodeSet = nodes.build(); + for (InternalNode node : nodeSet) { splits.add(new SystemSplit(tableHandle.getConnectorId(), tableHandle, node.getHostAndPort(), constraint)); } return new FixedSplitSource(splits.build()); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java b/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java index 9e4eb75851ec2..3f192bea26eb4 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/TaskCountEstimator.java @@ -14,8 +14,8 @@ package com.facebook.presto.cost; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.spi.Node; import javax.inject.Inject; @@ -35,7 +35,7 @@ public TaskCountEstimator(NodeSchedulerConfig nodeSchedulerConfig, InternalNodeM requireNonNull(nodeSchedulerConfig, "nodeSchedulerConfig is null"); requireNonNull(nodeManager, "nodeManager is null"); this.numberOfNodes = () -> { - Set activeNodes = nodeManager.getAllNodes().getActiveNodes(); + Set activeNodes = nodeManager.getAllNodes().getActiveNodes(); if (nodeSchedulerConfig.isIncludeCoordinator()) { return activeNodes.size(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java index f4e6f227d732b..e61f2b62f6374 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/LocationFactory.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.QueryId; import java.net.URI; @@ -26,7 +26,7 @@ public interface LocationFactory URI createLocalTaskLocation(TaskId taskId); - URI createTaskLocation(Node node, TaskId taskId); + URI createTaskLocation(InternalNode node, TaskId taskId); - URI createMemoryInfoLocation(Node node); + URI createMemoryInfoLocation(InternalNode node); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java index 4906dd941679a..0be7b5baee923 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java @@ -17,8 +17,8 @@ import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; @@ -43,7 +43,7 @@ public MemoryTrackingRemoteTaskFactory(RemoteTaskFactory remoteTaskFactory, Quer @Override public RemoteTask createRemoteTask(Session session, TaskId taskId, - Node node, + InternalNode node, PlanFragment fragment, Multimap initialSplits, OptionalInt totalPartitions, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java b/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java index d472c807c7c76..ac9c55657c120 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.util.FinalizerService; import com.google.common.collect.Sets; import io.airlift.log.Logger; @@ -33,7 +33,7 @@ public class NodeTaskMap { private static final Logger log = Logger.get(NodeTaskMap.class); - private final ConcurrentHashMap nodeTasksMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap nodeTasksMap = new ConcurrentHashMap<>(); private final FinalizerService finalizerService; @Inject @@ -42,22 +42,22 @@ public NodeTaskMap(FinalizerService finalizerService) this.finalizerService = requireNonNull(finalizerService, "finalizerService is null"); } - public void addTask(Node node, RemoteTask task) + public void addTask(InternalNode node, RemoteTask task) { createOrGetNodeTasks(node).addTask(task); } - public int getPartitionedSplitsOnNode(Node node) + public int getPartitionedSplitsOnNode(InternalNode node) { return createOrGetNodeTasks(node).getPartitionedSplitCount(); } - public PartitionedSplitCountTracker createPartitionedSplitCountTracker(Node node, TaskId taskId) + public PartitionedSplitCountTracker createPartitionedSplitCountTracker(InternalNode node, TaskId taskId) { return createOrGetNodeTasks(node).createPartitionedSplitCountTracker(taskId); } - private NodeTasks createOrGetNodeTasks(Node node) + private NodeTasks createOrGetNodeTasks(InternalNode node) { NodeTasks nodeTasks = nodeTasksMap.get(node); if (nodeTasks == null) { @@ -66,7 +66,7 @@ private NodeTasks createOrGetNodeTasks(Node node) return nodeTasks; } - private NodeTasks addNodeTask(Node node) + private NodeTasks addNodeTask(InternalNode node) { NodeTasks newNodeTasks = new NodeTasks(finalizerService); NodeTasks nodeTasks = nodeTasksMap.putIfAbsent(node, newNodeTasks); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java index c279974b9cbea..b6fbe0df63b93 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; @@ -29,7 +29,7 @@ public interface RemoteTaskFactory { RemoteTask createRemoteTask(Session session, TaskId taskId, - Node node, + InternalNode node, PlanFragment fragment, Multimap initialSplits, OptionalInt totalPartitions, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java index 51572cb028d03..409a39e977321 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java @@ -18,9 +18,9 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; import com.facebook.presto.failureDetector.FailureDetector; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.RemoteTransactionHandle; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.split.RemoteSplit; import com.facebook.presto.sql.planner.PlanFragment; @@ -80,7 +80,7 @@ public final class SqlStageExecution private final Map exchangeSources; - private final Map> tasks = new ConcurrentHashMap<>(); + private final Map> tasks = new ConcurrentHashMap<>(); @GuardedBy("this") private final AtomicInteger nextTaskId = new AtomicInteger(); @@ -359,7 +359,7 @@ public List getAllTasks() .collect(toImmutableList()); } - public synchronized Optional scheduleTask(Node node, int partition, OptionalInt totalPartitions) + public synchronized Optional scheduleTask(InternalNode node, int partition, OptionalInt totalPartitions) { requireNonNull(node, "node is null"); @@ -370,7 +370,7 @@ public synchronized Optional scheduleTask(Node node, int partition, return Optional.of(scheduleTask(node, new TaskId(stateMachine.getStageId(), partition), ImmutableMultimap.of(), totalPartitions)); } - public synchronized Set scheduleSplits(Node node, Multimap splits, Multimap noMoreSplitsNotification) + public synchronized Set scheduleSplits(InternalNode node, Multimap splits, Multimap noMoreSplitsNotification) { requireNonNull(node, "node is null"); requireNonNull(splits, "splits is null"); @@ -408,7 +408,7 @@ public synchronized Set scheduleSplits(Node node, Multimap sourceSplits, OptionalInt totalPartitions) + private synchronized RemoteTask scheduleTask(InternalNode node, TaskId taskId, Multimap sourceSplits, OptionalInt totalPartitions) { checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId); @@ -456,7 +456,7 @@ private synchronized RemoteTask scheduleTask(Node node, TaskId taskId, Multimap< return task; } - public Set getScheduledNodes() + public Set getScheduledNodes() { return ImmutableSet.copyOf(tasks.keySet()); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java index 87ef872edad78..e6cf72f41e2ab 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/BucketNodeMap.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import java.util.Optional; import java.util.function.ToIntFunction; @@ -32,13 +32,13 @@ public BucketNodeMap(ToIntFunction splitToBucket) public abstract int getBucketCount(); - public abstract Optional getAssignedNode(int bucketedId); + public abstract Optional getAssignedNode(int bucketedId); - public abstract void assignBucketToNode(int bucketedId, Node node); + public abstract void assignBucketToNode(int bucketedId, InternalNode node); public abstract boolean isDynamic(); - public final Optional getAssignedNode(Split split) + public final Optional getAssignedNode(Split split) { return getAssignedNode(splitToBucket.applyAsInt(split)); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java index 774ba6e23defd..3adaa9705d80c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/DynamicSplitPlacementPolicy.java @@ -14,8 +14,8 @@ package com.facebook.presto.execution.scheduler; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import java.util.List; import java.util.Set; @@ -48,7 +48,7 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return nodeSelector.allNodes(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java index 9e977edbe20ae..46858f41255b4 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedBucketNodeMap.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableList; import java.util.List; @@ -27,16 +27,16 @@ public class FixedBucketNodeMap extends BucketNodeMap { - private final List bucketToNode; + private final List bucketToNode; - public FixedBucketNodeMap(ToIntFunction splitToBucket, List bucketToNode) + public FixedBucketNodeMap(ToIntFunction splitToBucket, List bucketToNode) { super(splitToBucket); this.bucketToNode = ImmutableList.copyOf(requireNonNull(bucketToNode, "bucketToNode is null")); } @Override - public Optional getAssignedNode(int bucketedId) + public Optional getAssignedNode(int bucketedId) { return Optional.of(bucketToNode.get(bucketedId)); } @@ -48,7 +48,7 @@ public int getBucketCount() } @Override - public void assignBucketToNode(int bucketedId, Node node) + public void assignBucketToNode(int bucketedId, InternalNode node) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java index afafbe61179f0..f458e5cb57b4c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedCountScheduler.java @@ -15,7 +15,7 @@ import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.SqlStageExecution; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.google.common.annotations.VisibleForTesting; import java.util.List; @@ -31,13 +31,13 @@ public class FixedCountScheduler { public interface TaskScheduler { - Optional scheduleTask(Node node, int partition, OptionalInt totalPartitions); + Optional scheduleTask(InternalNode node, int partition, OptionalInt totalPartitions); } private final TaskScheduler taskScheduler; - private final List partitionToNode; + private final List partitionToNode; - public FixedCountScheduler(SqlStageExecution stage, List partitionToNode) + public FixedCountScheduler(SqlStageExecution stage, List partitionToNode) { requireNonNull(stage, "stage is null"); this.taskScheduler = stage::scheduleTask; @@ -45,7 +45,7 @@ public FixedCountScheduler(SqlStageExecution stage, List partitionToNode) } @VisibleForTesting - public FixedCountScheduler(TaskScheduler taskScheduler, List partitionToNode) + public FixedCountScheduler(TaskScheduler taskScheduler, List partitionToNode) { this.taskScheduler = requireNonNull(taskScheduler, "taskScheduler is null"); this.partitionToNode = requireNonNull(partitionToNode, "partitionToNode is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java index 82767b021bcfc..75646cbe56b79 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java @@ -20,9 +20,9 @@ import com.facebook.presto.execution.scheduler.group.DynamicLifespanScheduler; import com.facebook.presto.execution.scheduler.group.FixedLifespanScheduler; import com.facebook.presto.execution.scheduler.group.LifespanScheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.StageExecutionDescriptor; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.split.SplitSource; import com.facebook.presto.sql.planner.plan.PlanNodeId; @@ -57,7 +57,7 @@ public class FixedSourcePartitionedScheduler private static final Logger log = Logger.get(FixedSourcePartitionedScheduler.class); private final SqlStageExecution stage; - private final List nodes; + private final List nodes; private final List sourceSchedulers; private final List partitionHandles; private boolean scheduledTasks; @@ -68,7 +68,7 @@ public FixedSourcePartitionedScheduler( Map splitSources, StageExecutionDescriptor stageExecutionDescriptor, List schedulingOrder, - List nodes, + List nodes, BucketNodeMap bucketNodeMap, int splitBatchSize, OptionalInt concurrentLifespansPerTask, @@ -251,13 +251,13 @@ public static class BucketedSplitPlacementPolicy implements SplitPlacementPolicy { private final NodeSelector nodeSelector; - private final List allNodes; + private final List allNodes; private final BucketNodeMap bucketNodeMap; private final Supplier> remoteTasks; public BucketedSplitPlacementPolicy( NodeSelector nodeSelector, - List allNodes, + List allNodes, BucketNodeMap bucketNodeMap, Supplier> remoteTasks) { @@ -279,12 +279,12 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return allNodes; } - public Node getNodeForBucket(int bucketId) + public InternalNode getNodeForBucket(int bucketId) { return bucketNodeMap.getAssignedNode(bucketId).get(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java index cecd12fe1b033..023ea9771307a 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeAssignmentStats.java @@ -15,7 +15,7 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import java.util.HashMap; import java.util.List; @@ -27,8 +27,8 @@ public final class NodeAssignmentStats { private final NodeTaskMap nodeTaskMap; - private final Map assignmentCount = new HashMap<>(); - private final Map splitCountByNode = new HashMap<>(); + private final Map assignmentCount = new HashMap<>(); + private final Map splitCountByNode = new HashMap<>(); private final Map queuedSplitCountByNode = new HashMap<>(); public NodeAssignmentStats(NodeTaskMap nodeTaskMap, NodeMap nodeMap, List existingTasks) @@ -36,7 +36,7 @@ public NodeAssignmentStats(NodeTaskMap nodeTaskMap, NodeMap nodeMap, List x + y); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java index bba634b8e62ed..000221b9a4fd3 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.google.common.collect.SetMultimap; import java.net.InetAddress; @@ -22,14 +22,14 @@ public class NodeMap { - private final SetMultimap nodesByHostAndPort; - private final SetMultimap nodesByHost; - private final SetMultimap workersByNetworkPath; + private final SetMultimap nodesByHostAndPort; + private final SetMultimap nodesByHost; + private final SetMultimap workersByNetworkPath; private final Set coordinatorNodeIds; - public NodeMap(SetMultimap nodesByHostAndPort, - SetMultimap nodesByHost, - SetMultimap workersByNetworkPath, + public NodeMap(SetMultimap nodesByHostAndPort, + SetMultimap nodesByHost, + SetMultimap workersByNetworkPath, Set coordinatorNodeIds) { this.nodesByHostAndPort = nodesByHostAndPort; @@ -38,17 +38,17 @@ public NodeMap(SetMultimap nodesByHostAndPort, this.coordinatorNodeIds = coordinatorNodeIds; } - public SetMultimap getNodesByHostAndPort() + public SetMultimap getNodesByHostAndPort() { return nodesByHostAndPort; } - public SetMultimap getNodesByHost() + public SetMultimap getNodesByHost() { return nodesByHost; } - public SetMultimap getWorkersByNetworkPath() + public SetMultimap getWorkersByNetworkPath() { return workersByNetworkPath; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java index 20d018c36e775..e325a6b0949a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java @@ -16,10 +16,10 @@ import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.HashMultimap; @@ -125,11 +125,11 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) // this supplier is thread-safe. TODO: this logic should probably move to the scheduler since the choice of which node to run in should be // done as close to when the the split is about to be scheduled Supplier nodeMap = Suppliers.memoizeWithExpiration(() -> { - ImmutableSetMultimap.Builder byHostAndPort = ImmutableSetMultimap.builder(); - ImmutableSetMultimap.Builder byHost = ImmutableSetMultimap.builder(); - ImmutableSetMultimap.Builder workersByNetworkPath = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder byHostAndPort = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder byHost = ImmutableSetMultimap.builder(); + ImmutableSetMultimap.Builder workersByNetworkPath = ImmutableSetMultimap.builder(); - Set nodes; + Set nodes; if (connectorId != null) { nodes = nodeManager.getActiveConnectorNodes(connectorId); } @@ -138,10 +138,10 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) } Set coordinatorNodeIds = nodeManager.getCoordinators().stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .collect(toImmutableSet()); - for (Node node : nodes) { + for (InternalNode node : nodes) { if (useNetworkTopology && (includeCoordinator || !coordinatorNodeIds.contains(node.getNodeIdentifier()))) { NetworkLocation location = networkLocationCache.get(node.getHostAndPort()); for (int i = 0; i <= location.getSegments().size(); i++) { @@ -180,11 +180,11 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) } } - public static List selectNodes(int limit, ResettableRandomizedIterator candidates) + public static List selectNodes(int limit, ResettableRandomizedIterator candidates) { checkArgument(limit > 0, "limit must be at least 1"); - List selected = new ArrayList<>(min(limit, candidates.size())); + List selected = new ArrayList<>(min(limit, candidates.size())); while (selected.size() < limit && candidates.hasNext()) { selected.add(candidates.next()); } @@ -192,18 +192,18 @@ public static List selectNodes(int limit, ResettableRandomizedIterator randomizedNodes(NodeMap nodeMap, boolean includeCoordinator, Set excludedNodes) + public static ResettableRandomizedIterator randomizedNodes(NodeMap nodeMap, boolean includeCoordinator, Set excludedNodes) { - ImmutableList nodes = nodeMap.getNodesByHostAndPort().values().stream() + ImmutableList nodes = nodeMap.getNodesByHostAndPort().values().stream() .filter(node -> includeCoordinator || !nodeMap.getCoordinatorNodeIds().contains(node.getNodeIdentifier())) .filter(node -> !excludedNodes.contains(node)) .collect(toImmutableList()); return new ResettableRandomizedIterator<>(nodes); } - public static List selectExactNodes(NodeMap nodeMap, List hosts, boolean includeCoordinator) + public static List selectExactNodes(NodeMap nodeMap, List hosts, boolean includeCoordinator) { - Set chosen = new LinkedHashSet<>(); + Set chosen = new LinkedHashSet<>(); Set coordinatorIds = nodeMap.getCoordinatorNodeIds(); for (HostAddress host : hosts) { @@ -267,13 +267,13 @@ public static SplitPlacementResult selectDistributionNodes( List existingTasks, BucketNodeMap bucketNodeMap) { - Multimap assignments = HashMultimap.create(); + Multimap assignments = HashMultimap.create(); NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks); - Set blockedNodes = new HashSet<>(); + Set blockedNodes = new HashSet<>(); for (Split split : splits) { // node placement is forced by the bucket to node map - Node node = bucketNodeMap.getAssignedNode(split).get(); + InternalNode node = bucketNodeMap.getAssignedNode(split).get(); // if node is full, don't schedule now, which will push back on the scheduling of splits if (assignmentStats.getTotalSplitCount(node) < maxSplitsPerNode || @@ -295,7 +295,7 @@ public static int calculateLowWatermark(int maxPendingSplitsPerTask) return (int) Math.ceil(maxPendingSplitsPerTask / 2.0); } - public static ListenableFuture toWhenHasSplitQueueSpaceFuture(Set blockedNodes, List existingTasks, int spaceThreshold) + public static ListenableFuture toWhenHasSplitQueueSpaceFuture(Set blockedNodes, List existingTasks, int spaceThreshold) { if (blockedNodes.isEmpty()) { return immediateFuture(null); @@ -305,7 +305,7 @@ public static ListenableFuture toWhenHasSplitQueueSpaceFuture(Set block nodeToTaskMap.put(task.getNodeId(), task); } List> blockedFutures = blockedNodes.stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .map(nodeToTaskMap::get) .filter(Objects::nonNull) .map(remoteTask -> remoteTask.whenSplitQueueHasSpace(spaceThreshold)) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java index ae346530a44e3..b58f597313c1f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSelector.java @@ -14,8 +14,8 @@ package com.facebook.presto.execution.scheduler; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableSet; import java.util.List; @@ -25,16 +25,16 @@ public interface NodeSelector { void lockDownNodes(); - List allNodes(); + List allNodes(); - Node selectCurrentNode(); + InternalNode selectCurrentNode(); - default List selectRandomNodes(int limit) + default List selectRandomNodes(int limit) { return selectRandomNodes(limit, ImmutableSet.of()); } - List selectRandomNodes(int limit, Set excludedNodes); + List selectRandomNodes(int limit, Set excludedNodes); /** * Identifies the nodes for running the specified splits. diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java index 4b2ba9a56e290..393e4d91a7f2e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ScaledWriterScheduler.java @@ -16,7 +16,7 @@ import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.SqlStageExecution; import com.facebook.presto.execution.TaskStatus; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.SettableFuture; import io.airlift.units.DataSize; @@ -46,7 +46,7 @@ public class ScaledWriterScheduler private final NodeSelector nodeSelector; private final ScheduledExecutorService executor; private final long writerMinSizeBytes; - private final Set scheduledNodes = new HashSet<>(); + private final Set scheduledNodes = new HashSet<>(); private final AtomicBoolean done = new AtomicBoolean(); private volatile SettableFuture future = SettableFuture.create(); @@ -114,12 +114,12 @@ private List scheduleTasks(int count) return ImmutableList.of(); } - List nodes = nodeSelector.selectRandomNodes(count, scheduledNodes); + List nodes = nodeSelector.selectRandomNodes(count, scheduledNodes); checkCondition(!scheduledNodes.isEmpty() || !nodes.isEmpty(), NO_NODES_AVAILABLE, "No nodes available to run query"); ImmutableList.Builder tasks = ImmutableList.builder(); - for (Node node : nodes) { + for (InternalNode node : nodes) { Optional remoteTask = stage.scheduleTask(node, scheduledNodes.size(), OptionalInt.empty()); remoteTask.ifPresent(task -> { tasks.add(task); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java index 1b4d78588aaf3..47d5b22b9f653 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java @@ -15,9 +15,9 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -80,20 +80,20 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return ImmutableList.copyOf(nodeMap.get().get().getNodesByHostAndPort().values()); } @Override - public Node selectCurrentNode() + public InternalNode selectCurrentNode() { // TODO: this is a hack to force scheduling on the coordinator return nodeManager.getCurrentNode(); } @Override - public List selectRandomNodes(int limit, Set excludedNodes) + public List selectRandomNodes(int limit, Set excludedNodes) { return selectNodes(limit, randomizedNodes(nodeMap.get().get(), includeCoordinator, excludedNodes)); } @@ -101,17 +101,17 @@ public List selectRandomNodes(int limit, Set excludedNodes) @Override public SplitPlacementResult computeAssignments(Set splits, List existingTasks) { - Multimap assignment = HashMultimap.create(); + Multimap assignment = HashMultimap.create(); NodeMap nodeMap = this.nodeMap.get().get(); NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks); - ResettableRandomizedIterator randomCandidates = randomizedNodes(nodeMap, includeCoordinator, ImmutableSet.of()); - Set blockedExactNodes = new HashSet<>(); + ResettableRandomizedIterator randomCandidates = randomizedNodes(nodeMap, includeCoordinator, ImmutableSet.of()); + Set blockedExactNodes = new HashSet<>(); boolean splitWaitingForAnyNode = false; for (Split split : splits) { randomCandidates.reset(); - List candidateNodes; + List candidateNodes; if (!split.isRemotelyAccessible()) { candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); } @@ -123,10 +123,10 @@ public SplitPlacementResult computeAssignments(Set splits, List splits, List splitAssignment = ImmutableMultimap.of(); + Multimap splitAssignment = ImmutableMultimap.of(); if (!pendingSplits.isEmpty()) { if (!scheduleGroup.placementFuture.isDone()) { anyBlockedOnPlacements = true; @@ -272,11 +272,11 @@ else if (pendingSplits.isEmpty()) { } // if no new splits will be assigned, update state and attach completion event - Multimap noMoreSplitsNotification = ImmutableMultimap.of(); + Multimap noMoreSplitsNotification = ImmutableMultimap.of(); if (pendingSplits.isEmpty() && scheduleGroup.state == ScheduleGroupState.NO_MORE_SPLITS) { scheduleGroup.state = ScheduleGroupState.DONE; if (!lifespan.isTaskWide()) { - Node node = ((BucketedSplitPlacementPolicy) splitPlacementPolicy).getNodeForBucket(lifespan.getId()); + InternalNode node = ((BucketedSplitPlacementPolicy) splitPlacementPolicy).getNodeForBucket(lifespan.getId()); noMoreSplitsNotification = ImmutableMultimap.of(node, lifespan); } } @@ -429,15 +429,15 @@ public synchronized void notifyAllLifespansFinishedExecution() whenFinishedOrNewLifespanAdded.set(null); } - private Set assignSplits(Multimap splitAssignment, Multimap noMoreSplitsNotification) + private Set assignSplits(Multimap splitAssignment, Multimap noMoreSplitsNotification) { ImmutableSet.Builder newTasks = ImmutableSet.builder(); - ImmutableSet nodes = ImmutableSet.builder() + ImmutableSet nodes = ImmutableSet.builder() .addAll(splitAssignment.keySet()) .addAll(noMoreSplitsNotification.keySet()) .build(); - for (Node node : nodes) { + for (InternalNode node : nodes) { // source partitioned tasks can only receive broadcast data; otherwise it would have a different distribution ImmutableMultimap splits = ImmutableMultimap.builder() .putAll(partitionedNode, splitAssignment.get(node)) @@ -463,7 +463,7 @@ private Set finalizeTaskCreationIfNecessary() splitPlacementPolicy.lockDownNodes(); - Set scheduledNodes = stage.getScheduledNodes(); + Set scheduledNodes = stage.getScheduledNodes(); Set newTasks = splitPlacementPolicy.allNodes().stream() .filter(node -> !scheduledNodes.contains(node)) .flatMap(node -> stage.scheduleSplits(node, ImmutableMultimap.of(), ImmutableMultimap.of()).stream()) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java index 8c33676d5207f..9e8a9bf718791 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementPolicy.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import java.util.List; import java.util.Set; @@ -25,5 +25,5 @@ public interface SplitPlacementPolicy void lockDownNodes(); - List allNodes(); + List allNodes(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java index 0ce5fb1e0a8a3..2399b694fe86d 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitPlacementResult.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; @@ -23,9 +23,9 @@ public final class SplitPlacementResult { private final ListenableFuture blocked; - private final Multimap assignments; + private final Multimap assignments; - public SplitPlacementResult(ListenableFuture blocked, Multimap assignments) + public SplitPlacementResult(ListenableFuture blocked, Multimap assignments) { this.blocked = requireNonNull(blocked, "blocked is null"); this.assignments = requireNonNull(assignments, "assignments is null"); @@ -36,7 +36,7 @@ public ListenableFuture getBlocked() return blocked; } - public Multimap getAssignments() + public Multimap getAssignments() { return assignments; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index 09e6b7eae2a70..6df2db1a0d2e5 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -31,7 +31,7 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.failureDetector.FailureDetector; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.split.SplitSource; @@ -445,7 +445,7 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { } BucketNodeMap bucketNodeMap; - List stageNodeList; + List stageNodeList; if (plan.getFragment().getRemoteSourceNodes().stream().allMatch(node -> node.getExchangeType() == REPLICATE)) { // no non-replicated remote source boolean dynamicLifespanSchedule = plan.getFragment().getStageExecutionDescriptor().isDynamicLifespanSchedule(); @@ -487,7 +487,7 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { else { // all sources are remote NodePartitionMap nodePartitionMap = partitioningCache.apply(plan.getFragment().getPartitioning()); - List partitionToNode = nodePartitionMap.getPartitionToNode(); + List partitionToNode = nodePartitionMap.getPartitionToNode(); // todo this should asynchronously wait a standard timeout period before failing checkCondition(!partitionToNode.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available"); stageSchedulers.put(stageId, new FixedCountScheduler(stage, partitionToNode)); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java index 34c94fee1a7d2..0b4a1fd1b2122 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java @@ -15,10 +15,10 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -94,20 +94,20 @@ public void lockDownNodes() } @Override - public List allNodes() + public List allNodes() { return ImmutableList.copyOf(nodeMap.get().get().getNodesByHostAndPort().values()); } @Override - public Node selectCurrentNode() + public InternalNode selectCurrentNode() { // TODO: this is a hack to force scheduling on the coordinator return nodeManager.getCurrentNode(); } @Override - public List selectRandomNodes(int limit, Set excludedNodes) + public List selectRandomNodes(int limit, Set excludedNodes) { return selectNodes(limit, randomizedNodes(nodeMap.get().get(), includeCoordinator, excludedNodes)); } @@ -116,21 +116,21 @@ public List selectRandomNodes(int limit, Set excludedNodes) public SplitPlacementResult computeAssignments(Set splits, List existingTasks) { NodeMap nodeMap = this.nodeMap.get().get(); - Multimap assignment = HashMultimap.create(); + Multimap assignment = HashMultimap.create(); NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks); int[] topologicCounters = new int[topologicalSplitCounters.size()]; Set filledLocations = new HashSet<>(); - Set blockedExactNodes = new HashSet<>(); + Set blockedExactNodes = new HashSet<>(); boolean splitWaitingForAnyNode = false; for (Split split : splits) { if (!split.isRemotelyAccessible()) { - List candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); + List candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); if (candidateNodes.isEmpty()) { log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getNodesByHost().keys()); throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query"); } - Node chosenNode = bestNodeSplitCount(candidateNodes.iterator(), minCandidates, maxPendingSplitsPerTask, assignmentStats); + InternalNode chosenNode = bestNodeSplitCount(candidateNodes.iterator(), minCandidates, maxPendingSplitsPerTask, assignmentStats); if (chosenNode != null) { assignment.put(chosenNode, split); assignmentStats.addAssignedSplit(chosenNode); @@ -142,7 +142,7 @@ else if (!splitWaitingForAnyNode) { continue; } - Node chosenNode = null; + InternalNode chosenNode = null; int depth = networkLocationSegmentNames.size(); int chosenDepth = 0; Set locations = new HashSet<>(); @@ -166,7 +166,7 @@ else if (!splitWaitingForAnyNode) { if (filledLocations.contains(location)) { continue; } - Set nodes = nodeMap.getWorkersByNetworkPath().get(location); + Set nodes = nodeMap.getWorkersByNetworkPath().get(location); chosenNode = bestNodeSplitCount(new ResettableRandomizedIterator<>(nodes), minCandidates, calculateMaxPendingSplits(i, depth), assignmentStats); if (chosenNode != null) { chosenDepth = i; @@ -224,14 +224,14 @@ public SplitPlacementResult computeAssignments(Set splits, List candidates, int minCandidatesWhenFull, int maxPendingSplitsPerTask, NodeAssignmentStats assignmentStats) + private InternalNode bestNodeSplitCount(Iterator candidates, int minCandidatesWhenFull, int maxPendingSplitsPerTask, NodeAssignmentStats assignmentStats) { - Node bestQueueNotFull = null; + InternalNode bestQueueNotFull = null; int min = Integer.MAX_VALUE; int fullCandidatesConsidered = 0; while (candidates.hasNext() && (fullCandidatesConsidered < minCandidatesWhenFull || bestQueueNotFull == null)) { - Node node = candidates.next(); + InternalNode node = candidates.next(); if (assignmentStats.getTotalSplitCount(node) < maxSplitsPerNode) { return node; } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java index 4bfb13ca33bb4..69113959c5bd2 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicBucketNodeMap.java @@ -14,8 +14,8 @@ package com.facebook.presto.execution.scheduler.group; import com.facebook.presto.execution.scheduler.BucketNodeMap; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -30,7 +30,7 @@ public class DynamicBucketNodeMap extends BucketNodeMap { private final int bucketCount; - private final Int2ObjectMap bucketToNode = new Int2ObjectOpenHashMap<>(); + private final Int2ObjectMap bucketToNode = new Int2ObjectOpenHashMap<>(); public DynamicBucketNodeMap(ToIntFunction splitToBucket, int bucketCount) { @@ -40,7 +40,7 @@ public DynamicBucketNodeMap(ToIntFunction splitToBucket, int bucketCount) } @Override - public Optional getAssignedNode(int bucketedId) + public Optional getAssignedNode(int bucketedId) { return Optional.ofNullable(bucketToNode.get(bucketedId)); } @@ -52,7 +52,7 @@ public int getBucketCount() } @Override - public void assignBucketToNode(int bucketedId, Node node) + public void assignBucketToNode(int bucketedId, InternalNode node) { checkArgument(bucketedId >= 0 && bucketedId < bucketCount); requireNonNull(node, "node is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java index bc54f33c51b2c..4b51d34c7260b 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/DynamicLifespanScheduler.java @@ -16,7 +16,7 @@ import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.scheduler.SourceScheduler; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.SettableFuture; @@ -43,7 +43,7 @@ public class DynamicLifespanScheduler implements LifespanScheduler { private final BucketNodeMap bucketNodeMap; - private final List allNodes; + private final List allNodes; private final List partitionHandles; private final OptionalInt concurrentLifespansPerTask; @@ -61,7 +61,7 @@ public class DynamicLifespanScheduler @GuardedBy("this") private int totalLifespanExecutionFinished; - public DynamicLifespanScheduler(BucketNodeMap bucketNodeMap, List allNodes, List partitionHandles, OptionalInt concurrentLifespansPerTask) + public DynamicLifespanScheduler(BucketNodeMap bucketNodeMap, List allNodes, List partitionHandles, OptionalInt concurrentLifespansPerTask) { this.bucketNodeMap = requireNonNull(bucketNodeMap, "bucketNodeMap is null"); this.allNodes = requireNonNull(allNodes, "allNodes is null"); @@ -136,7 +136,7 @@ public SettableFuture schedule(SourceScheduler scheduler) } int driverGroupId = driverGroups.nextInt(); - Node nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new); + InternalNode nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new); bucketNodeMap.assignBucketToNode(driverGroupId, nodeForCompletedDriverGroup); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java index bb6dd83e72744..d80a9f79f7aec 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/group/FixedLifespanScheduler.java @@ -16,7 +16,7 @@ import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.scheduler.SourceScheduler; -import com.facebook.presto.spi.Node; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.SettableFuture; @@ -47,8 +47,8 @@ public class FixedLifespanScheduler implements LifespanScheduler { - private final Int2ObjectMap driverGroupToNodeMap; - private final Map nodeToDriverGroupsMap; + private final Int2ObjectMap driverGroupToNodeMap; + private final Map nodeToDriverGroupsMap; private final List partitionHandles; private final OptionalInt concurrentLifespansPerTask; @@ -64,10 +64,10 @@ public FixedLifespanScheduler(BucketNodeMap bucketNodeMap, List nodeToDriverGroupMap = new HashMap<>(); - Int2ObjectMap driverGroupToNodeMap = new Int2ObjectOpenHashMap<>(); + Map nodeToDriverGroupMap = new HashMap<>(); + Int2ObjectMap driverGroupToNodeMap = new Int2ObjectOpenHashMap<>(); for (int bucket = 0; bucket < bucketNodeMap.getBucketCount(); bucket++) { - Node node = bucketNodeMap.getAssignedNode(bucket).get(); + InternalNode node = bucketNodeMap.getAssignedNode(bucket).get(); nodeToDriverGroupMap.computeIfAbsent(node, key -> new IntArrayList()).add(bucket); driverGroupToNodeMap.put(bucket, node); } @@ -87,7 +87,7 @@ public void scheduleInitial(SourceScheduler scheduler) checkState(!initialScheduled); initialScheduled = true; - for (Map.Entry entry : nodeToDriverGroupsMap.entrySet()) { + for (Map.Entry entry : nodeToDriverGroupsMap.entrySet()) { IntListIterator driverGroupsIterator = entry.getValue(); int driverGroupsScheduled = 0; while (driverGroupsIterator.hasNext()) { diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java index d0600b34778b2..b9d9749d0c676 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java @@ -18,13 +18,13 @@ import com.facebook.presto.execution.QueryIdGenerator; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.ServerConfig; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; @@ -478,14 +478,14 @@ private synchronized boolean allAssignmentsHavePropagated(Iterable builder = ImmutableSet.builder(); - Set aliveNodes = builder + ImmutableSet.Builder builder = ImmutableSet.builder(); + Set aliveNodes = builder .addAll(nodeManager.getNodes(ACTIVE)) .addAll(nodeManager.getNodes(SHUTTING_DOWN)) .build(); ImmutableSet aliveNodeIds = aliveNodes.stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .collect(toImmutableSet()); // Remove nodes that don't exist anymore @@ -494,7 +494,7 @@ private synchronized void updateNodes(MemoryPoolAssignmentsRequest assignments) nodes.keySet().removeAll(deadNodes); // Add new nodes - for (Node node : aliveNodes) { + for (InternalNode node : aliveNodes) { if (!nodes.containsKey(node.getNodeIdentifier())) { nodes.put( node.getNodeIdentifier(), diff --git a/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java b/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java index 7daf1beebccbf..df98ce5d8a31e 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.memory; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.Node; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import io.airlift.http.client.HttpClient; @@ -54,7 +54,7 @@ public class RemoteNodeMemory { private static final Logger log = Logger.get(RemoteNodeMemory.class); - private final Node node; + private final InternalNode node; private final HttpClient httpClient; private final URI memoryInfoUri; private final Codec memoryInfoCodec; @@ -67,7 +67,7 @@ public class RemoteNodeMemory private final boolean isBinaryTransportEnabled; public RemoteNodeMemory( - Node node, + InternalNode node, HttpClient httpClient, Codec memoryInfoCodec, Codec assignmentsRequestCodec, @@ -92,7 +92,7 @@ public Optional getInfo() return memoryInfo.get(); } - public Node getNode() + public InternalNode getNode() { return node; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java b/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java index 4a449ccf35279..d0f9240bbcc01 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/AllNodes.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.metadata; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableSet; import java.util.Objects; @@ -23,12 +22,12 @@ public class AllNodes { - private final Set activeNodes; - private final Set inactiveNodes; - private final Set shuttingDownNodes; - private final Set activeCoordinators; + private final Set activeNodes; + private final Set inactiveNodes; + private final Set shuttingDownNodes; + private final Set activeCoordinators; - public AllNodes(Set activeNodes, Set inactiveNodes, Set shuttingDownNodes, Set activeCoordinators) + public AllNodes(Set activeNodes, Set inactiveNodes, Set shuttingDownNodes, Set activeCoordinators) { this.activeNodes = ImmutableSet.copyOf(requireNonNull(activeNodes, "activeNodes is null")); this.inactiveNodes = ImmutableSet.copyOf(requireNonNull(inactiveNodes, "inactiveNodes is null")); @@ -36,22 +35,22 @@ public AllNodes(Set activeNodes, Set inactiveNodes, Set shutti this.activeCoordinators = ImmutableSet.copyOf(requireNonNull(activeCoordinators, "activeCoordinators is null")); } - public Set getActiveNodes() + public Set getActiveNodes() { return activeNodes; } - public Set getInactiveNodes() + public Set getInactiveNodes() { return inactiveNodes; } - public Set getShuttingDownNodes() + public Set getShuttingDownNodes() { return shuttingDownNodes; } - public Set getActiveCoordinators() + public Set getActiveCoordinators() { return activeCoordinators; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index 33534f8930623..6bd45c3c0fde7 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -18,7 +18,6 @@ import com.facebook.presto.connector.system.GlobalSystemConnector; import com.facebook.presto.failureDetector.FailureDetector; import com.facebook.presto.server.InternalCommunicationConfig; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -84,13 +83,13 @@ public final class DiscoveryNodeManager private final InternalNode currentNode; @GuardedBy("this") - private SetMultimap activeNodesByConnectorId; + private SetMultimap activeNodesByConnectorId; @GuardedBy("this") private AllNodes allNodes; @GuardedBy("this") - private Set coordinators; + private Set coordinators; @GuardedBy("this") private final List> listeners = new ArrayList<>(); @@ -159,13 +158,13 @@ public void startPollingNodeStates() private void pollWorkers() { AllNodes allNodes = getAllNodes(); - Set aliveNodes = ImmutableSet.builder() + Set aliveNodes = ImmutableSet.builder() .addAll(allNodes.getActiveNodes()) .addAll(allNodes.getShuttingDownNodes()) .build(); ImmutableSet aliveNodeIds = aliveNodes.stream() - .map(Node::getNodeIdentifier) + .map(InternalNode::getNodeIdentifier) .collect(toImmutableSet()); // Remove nodes that don't exist anymore @@ -174,7 +173,7 @@ private void pollWorkers() nodeStates.keySet().removeAll(deadNodes); // Add new nodes - for (Node node : aliveNodes) { + for (InternalNode node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); } @@ -206,11 +205,11 @@ private synchronized void refreshNodesInternal() .filter(service -> !failureDetector.getFailed().contains(service)) .collect(toImmutableSet()); - ImmutableSet.Builder activeNodesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder inactiveNodesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder shuttingDownNodesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder coordinatorsBuilder = ImmutableSet.builder(); - ImmutableSetMultimap.Builder byConnectorIdBuilder = ImmutableSetMultimap.builder(); + ImmutableSet.Builder activeNodesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder inactiveNodesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder shuttingDownNodesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder coordinatorsBuilder = ImmutableSet.builder(); + ImmutableSetMultimap.Builder byConnectorIdBuilder = ImmutableSetMultimap.builder(); for (ServiceDescriptor service : services) { URI uri = getHttpUri(service, httpsRequired); @@ -253,8 +252,8 @@ private synchronized void refreshNodesInternal() if (allNodes != null) { // log node that are no longer active (but not shutting down) - SetView missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); - for (Node missingNode : missingNodes) { + SetView missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); + for (InternalNode missingNode : missingNodes) { log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHostAndPort()); } } @@ -323,7 +322,7 @@ public int getShuttingDownNodeCount() } @Override - public Set getNodes(NodeState state) + public Set getNodes(NodeState state) { switch (state) { case ACTIVE: @@ -338,19 +337,19 @@ public Set getNodes(NodeState state) } @Override - public synchronized Set getActiveConnectorNodes(ConnectorId connectorId) + public synchronized Set getActiveConnectorNodes(ConnectorId connectorId) { return activeNodesByConnectorId.get(connectorId); } @Override - public Node getCurrentNode() + public InternalNode getCurrentNode() { return currentNode; } @Override - public synchronized Set getCoordinators() + public synchronized Set getCoordinators() { return coordinators; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java index fa90fad75b8db..9a550b207f771 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InMemoryNodeManager.java @@ -15,7 +15,6 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.connector.ConnectorId; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; @@ -37,8 +36,8 @@ public class InMemoryNodeManager implements InternalNodeManager { - private final Node localNode; - private final SetMultimap remoteNodes = Multimaps.synchronizedSetMultimap(HashMultimap.create()); + private final InternalNode localNode; + private final SetMultimap remoteNodes = Multimaps.synchronizedSetMultimap(HashMultimap.create()); @GuardedBy("this") private final List> listeners = new ArrayList<>(); @@ -59,12 +58,12 @@ public void addCurrentNodeConnector(ConnectorId connectorId) addNode(connectorId, localNode); } - public void addNode(ConnectorId connectorId, Node... nodes) + public void addNode(ConnectorId connectorId, InternalNode... nodes) { addNode(connectorId, ImmutableList.copyOf(nodes)); } - public void addNode(ConnectorId connectorId, Iterable nodes) + public void addNode(ConnectorId connectorId, Iterable nodes) { remoteNodes.putAll(connectorId, nodes); @@ -77,7 +76,7 @@ public void addNode(ConnectorId connectorId, Iterable nodes) } @Override - public Set getNodes(NodeState state) + public Set getNodes(NodeState state) { switch (state) { case ACTIVE: @@ -92,7 +91,7 @@ public Set getNodes(NodeState state) } @Override - public Set getActiveConnectorNodes(ConnectorId connectorId) + public Set getActiveConnectorNodes(ConnectorId connectorId) { return ImmutableSet.copyOf(remoteNodes.get(connectorId)); } @@ -100,17 +99,17 @@ public Set getActiveConnectorNodes(ConnectorId connectorId) @Override public AllNodes getAllNodes() { - return new AllNodes(ImmutableSet.builder().add(localNode).addAll(remoteNodes.values()).build(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(localNode)); + return new AllNodes(ImmutableSet.builder().add(localNode).addAll(remoteNodes.values()).build(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(localNode)); } @Override - public Node getCurrentNode() + public InternalNode getCurrentNode() { return localNode; } @Override - public Set getCoordinators() + public Set getCoordinators() { // always use localNode as coordinator return ImmutableSet.of(localNode); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java index a0d13c6f2754d..a6b63b5ee6a38 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNodeManager.java @@ -14,7 +14,6 @@ package com.facebook.presto.metadata; import com.facebook.presto.connector.ConnectorId; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import java.util.Set; @@ -22,13 +21,13 @@ public interface InternalNodeManager { - Set getNodes(NodeState state); + Set getNodes(NodeState state); - Set getActiveConnectorNodes(ConnectorId connectorId); + Set getActiveConnectorNodes(ConnectorId connectorId); - Node getCurrentNode(); + InternalNode getCurrentNode(); - Set getCoordinators(); + Set getCoordinators(); AllNodes getAllNodes(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java index b8095b3422633..7672f84ff57b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeState; import io.airlift.http.client.HttpClient; import io.airlift.http.client.Request; @@ -68,8 +68,8 @@ public Response getThreads(@PathParam("nodeId") String nodeId) private Response proxyJsonResponse(String nodeId, String workerPath) { - Set nodes = nodeManager.getNodes(NodeState.ACTIVE); - Node node = nodes.stream() + Set nodes = nodeManager.getNodes(NodeState.ACTIVE); + InternalNode node = nodes.stream() .filter(n -> n.getNodeIdentifier().equals(nodeId)) .findFirst() .orElseThrow(() -> new WebApplicationException(NOT_FOUND)); diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java index 878fb511508a8..6c88bed17f740 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java @@ -16,9 +16,9 @@ import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.StageId; import com.facebook.presto.execution.TaskId; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.server.InternalCommunicationConfig; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import io.airlift.http.server.HttpServerInfo; @@ -74,7 +74,7 @@ public URI createLocalTaskLocation(TaskId taskId) } @Override - public URI createTaskLocation(Node node, TaskId taskId) + public URI createTaskLocation(InternalNode node, TaskId taskId) { requireNonNull(node, "node is null"); requireNonNull(taskId, "taskId is null"); @@ -85,7 +85,7 @@ public URI createTaskLocation(Node node, TaskId taskId) } @Override - public URI createMemoryInfoLocation(Node node) + public URI createMemoryInfoLocation(InternalNode node) { requireNonNull(node, "node is null"); return uriBuilderFrom(node.getHttpUri()) diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java index 23cd8bbbf8329..7c9936d8a3d38 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java @@ -24,13 +24,13 @@ import com.facebook.presto.execution.TaskManagerConfig; import com.facebook.presto.execution.TaskStatus; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.ForScheduler; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.TaskUpdateRequest; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; @@ -153,7 +153,7 @@ public void stop() @Override public RemoteTask createRemoteTask(Session session, TaskId taskId, - Node node, + InternalNode node, PlanFragment fragment, Multimap initialSplits, OptionalInt totalPartitions, diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index 13a0bab62749b..66d68f7f5e42f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -27,6 +27,7 @@ import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.metadata.AllNodes; import com.facebook.presto.metadata.CatalogManager; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.security.AccessControl; @@ -37,7 +38,6 @@ import com.facebook.presto.server.ShutdownAction; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.server.smile.SmileModule; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.QueryId; import com.facebook.presto.split.PageSourceManager; @@ -493,7 +493,7 @@ public final AllNodes refreshNodes() return nodeManager.getAllNodes(); } - public Set getActiveNodesWithConnector(ConnectorId connectorId) + public Set getActiveNodesWithConnector(ConnectorId connectorId) { return nodeManager.getActiveConnectorNodes(connectorId); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java index b2f59e3c70b79..6e3f05eb4e002 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitionMap.java @@ -15,8 +15,8 @@ import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.scheduler.FixedBucketNodeMap; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableList; import java.util.List; @@ -37,25 +37,25 @@ // in the above case, as the co-existence of BucketNodeMap and NodePartitionMap is confusing. public class NodePartitionMap { - private final List partitionToNode; + private final List partitionToNode; private final int[] bucketToPartition; private final ToIntFunction splitToBucket; - public NodePartitionMap(List partitionToNode, ToIntFunction splitToBucket) + public NodePartitionMap(List partitionToNode, ToIntFunction splitToBucket) { this.partitionToNode = ImmutableList.copyOf(requireNonNull(partitionToNode, "partitionToNode is null")); this.bucketToPartition = IntStream.range(0, partitionToNode.size()).toArray(); this.splitToBucket = requireNonNull(splitToBucket, "splitToBucket is null"); } - public NodePartitionMap(List partitionToNode, int[] bucketToPartition, ToIntFunction splitToBucket) + public NodePartitionMap(List partitionToNode, int[] bucketToPartition, ToIntFunction splitToBucket) { this.bucketToPartition = requireNonNull(bucketToPartition, "bucketToPartition is null"); this.partitionToNode = ImmutableList.copyOf(requireNonNull(partitionToNode, "partitionToNode is null")); this.splitToBucket = requireNonNull(splitToBucket, "splitToBucket is null"); } - public List getPartitionToNode() + public List getPartitionToNode() { return partitionToNode; } @@ -65,7 +65,7 @@ public int[] getBucketToPartition() return bucketToPartition; } - public Node getNode(Split split) + public InternalNode getNode(Split split) { int bucket = splitToBucket.applyAsInt(split); int partition = bucketToPartition[bucket]; @@ -74,7 +74,7 @@ public Node getNode(Split split) public BucketNodeMap asBucketNodeMap() { - ImmutableList.Builder bucketToNode = ImmutableList.builder(); + ImmutableList.Builder bucketToNode = ImmutableList.builder(); for (int partition : bucketToPartition) { bucketToNode.add(partitionToNode.get(partition)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java index 888b1856f5bfc..fee4bced50184 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java @@ -19,12 +19,12 @@ import com.facebook.presto.execution.scheduler.FixedBucketNodeMap; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.group.DynamicBucketNodeMap; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.BucketPartitionFunction; import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.spi.BucketFunction; import com.facebook.presto.spi.ConnectorSplit; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.connector.ConnectorBucketNodeMap; import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; @@ -138,9 +138,9 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand // safety check for crazy partitioning checkArgument(connectorBucketNodeMap.getBucketCount() < 1_000_000, "Too many buckets in partitioning: %s", connectorBucketNodeMap.getBucketCount()); - List bucketToNode; + List bucketToNode; if (connectorBucketNodeMap.hasFixedMapping()) { - bucketToNode = connectorBucketNodeMap.getFixedMapping(); + bucketToNode = getFixedMapping(connectorBucketNodeMap); } else { bucketToNode = createArbitraryBucketToNode( @@ -149,10 +149,10 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand } int[] bucketToPartition = new int[connectorBucketNodeMap.getBucketCount()]; - BiMap nodeToPartition = HashBiMap.create(); + BiMap nodeToPartition = HashBiMap.create(); int nextPartitionId = 0; for (int bucket = 0; bucket < bucketToNode.size(); bucket++) { - Node node = bucketToNode.get(bucket); + InternalNode node = bucketToNode.get(bucket); Integer partitionId = nodeToPartition.get(node); if (partitionId == null) { partitionId = nextPartitionId++; @@ -161,7 +161,7 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand bucketToPartition[bucket] = partitionId; } - List partitionToNode = IntStream.range(0, nodeToPartition.size()) + List partitionToNode = IntStream.range(0, nodeToPartition.size()) .mapToObj(partitionId -> nodeToPartition.inverse().get(partitionId)) .collect(toImmutableList()); @@ -173,7 +173,7 @@ public BucketNodeMap getBucketNodeMap(Session session, PartitioningHandle partit ConnectorBucketNodeMap connectorBucketNodeMap = getConnectorBucketNodeMap(session, partitioningHandle); if (connectorBucketNodeMap.hasFixedMapping()) { - return new FixedBucketNodeMap(getSplitToBucket(session, partitioningHandle), connectorBucketNodeMap.getFixedMapping()); + return new FixedBucketNodeMap(getSplitToBucket(session, partitioningHandle), getFixedMapping(connectorBucketNodeMap)); } if (preferDynamic) { @@ -187,6 +187,13 @@ public BucketNodeMap getBucketNodeMap(Session session, PartitioningHandle partit connectorBucketNodeMap.getBucketCount())); } + private static List getFixedMapping(ConnectorBucketNodeMap connectorBucketNodeMap) + { + return connectorBucketNodeMap.getFixedMapping().stream() + .map(InternalNode.class::cast) + .collect(toImmutableList()); + } + private ConnectorBucketNodeMap getConnectorBucketNodeMap(Session session, PartitioningHandle partitioningHandle) { checkArgument(!(partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle)); @@ -229,7 +236,7 @@ private ToIntFunction getSplitToBucket(Session session, PartitioningHandl }; } - private static List createArbitraryBucketToNode(List nodes, int bucketCount) + private static List createArbitraryBucketToNode(List nodes, int bucketCount) { return cyclingShuffledStream(nodes) .limit(bucketCount) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java index 0f468b755131a..fe273ba2e729c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java @@ -16,13 +16,13 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.NodeSelector; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.BucketPartitionFunction; import com.facebook.presto.operator.HashGenerator; import com.facebook.presto.operator.InterpretedHashGenerator; import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.operator.PrecomputedHashGenerator; import com.facebook.presto.spi.BucketFunction; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; import com.facebook.presto.spi.type.Type; @@ -139,7 +139,7 @@ public String toString() public NodePartitionMap getNodePartitionMap(Session session, NodeScheduler nodeScheduler) { NodeSelector nodeSelector = nodeScheduler.createNodeSelector(null); - List nodes; + List nodes; if (partitioning == SystemPartitioning.COORDINATOR_ONLY) { nodes = ImmutableList.of(nodeSelector.selectCurrentNode()); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java index ec3eae93d7c07..810d1352005a7 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java @@ -27,7 +27,6 @@ import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.util.FinalizerService; @@ -100,8 +99,8 @@ public Object benchmark(BenchmarkData data) Iterator splits = data.getSplits().iterator(); Set batch = new HashSet<>(); while (splits.hasNext() || !batch.isEmpty()) { - Multimap assignments = data.getNodeSelector().computeAssignments(batch, remoteTasks).getAssignments(); - for (Node node : assignments.keySet()) { + Multimap assignments = data.getNodeSelector().computeAssignments(batch, remoteTasks).getAssignments(); + for (InternalNode node : assignments.keySet()) { MockRemoteTaskFactory.MockRemoteTask remoteTask = data.getTaskMap().get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -134,7 +133,7 @@ public static class BenchmarkData private FinalizerService finalizerService = new FinalizerService(); private NodeSelector nodeSelector; - private Map taskMap = new HashMap<>(); + private Map taskMap = new HashMap<>(); private List splits = new ArrayList<>(); @Setup @@ -145,16 +144,16 @@ public void setup() finalizerService.start(); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); - ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); for (int i = 0; i < NODES; i++) { nodeBuilder.add(new InternalNode("node" + i, URI.create("http://" + addressForHost(i).getHostText()), NodeVersion.UNKNOWN, false)); } - List nodes = nodeBuilder.build(); + List nodes = nodeBuilder.build(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory( newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s")), newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s"))); for (int i = 0; i < nodes.size(); i++) { - Node node = nodes.get(i); + InternalNode node = nodes.get(i); ImmutableList.Builder initialSplits = ImmutableList.builder(); for (int j = 0; j < MAX_SPLITS_PER_NODE + MAX_PENDING_SPLITS_PER_TASK_PER_NODE; j++) { initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(i))); @@ -209,7 +208,7 @@ private NetworkTopology getNetworkTopology() return topology; } - public Map getTaskMap() + public Map getTaskMap() { return taskMap; } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index 747473c49ea66..e5cbe71b92451 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -23,12 +23,12 @@ import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.QueryContext; import com.facebook.presto.memory.context.SimpleLocalMemoryContext; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.operator.TaskContext; import com.facebook.presto.operator.TaskStats; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spiller.SpillSpaceTracker; import com.facebook.presto.sql.planner.Partitioning; @@ -102,7 +102,7 @@ public MockRemoteTaskFactory(Executor executor, ScheduledExecutorService schedul this.scheduledExecutor = scheduledExecutor; } - public MockRemoteTask createTableScanTask(TaskId taskId, Node newNode, List splits, PartitionedSplitCountTracker partitionedSplitCountTracker) + public MockRemoteTask createTableScanTask(TaskId taskId, InternalNode newNode, List splits, PartitionedSplitCountTracker partitionedSplitCountTracker) { Symbol symbol = new Symbol("column"); PlanNodeId sourceId = new PlanNodeId("sourceId"); @@ -132,7 +132,7 @@ public MockRemoteTask createTableScanTask(TaskId taskId, Node newNode, List initialSplits, OptionalInt totalPartitions, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index 052042f611870..65a0aebc1156e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -27,7 +27,6 @@ import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Node; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.util.FinalizerService; @@ -73,7 +72,7 @@ public class TestNodeScheduler private NodeTaskMap nodeTaskMap; private InMemoryNodeManager nodeManager; private NodeSelector nodeSelector; - private Map taskMap; + private Map taskMap; private ExecutorService remoteTaskExecutor; private ScheduledExecutorService remoteTaskScheduledExecutor; @@ -84,11 +83,11 @@ public void setUp() nodeTaskMap = new NodeTaskMap(finalizerService); nodeManager = new InMemoryNodeManager(); - ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); nodeBuilder.add(new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); - ImmutableList nodes = nodeBuilder.build(); + ImmutableList nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(20) @@ -119,7 +118,7 @@ public void testScheduleLocal() Split split = new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitLocal()); Set splits = ImmutableSet.of(split); - Map.Entry assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments().entries()); + Map.Entry assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments().entries()); assertEquals(assignment.getKey().getHostAndPort(), split.getAddresses().get(0)); assertEquals(assignment.getValue(), split); } @@ -132,15 +131,15 @@ public void testTopologyAwareScheduling() NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); - ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); nodeBuilder.add(new InternalNode("node1", URI.create("http://host1.rack1:11"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new InternalNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new InternalNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false)); - ImmutableList nodes = nodeBuilder.build(); + ImmutableList nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); // contents of taskMap indicate the node-task map for the current stage - Map taskMap = new HashMap<>(); + Map taskMap = new HashMap<>(); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(25) .setIncludeCoordinator(false) @@ -171,10 +170,10 @@ public NetworkLocation get(HostAddress host) nonRackLocalBuilder.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromParts("data.other_rack", 1)))); } Set nonRackLocalSplits = nonRackLocalBuilder.build(); - Multimap assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); int task = 0; - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { TaskId taskId = new TaskId("test", 1, task); task++; MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); @@ -185,7 +184,7 @@ public NetworkLocation get(HostAddress host) // Continue assigning to fill up part of the queue nonRackLocalSplits = Sets.difference(nonRackLocalSplits, new HashSet<>(assignments.values())); assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments(); - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { RemoteTask remoteTask = taskMap.get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -206,7 +205,7 @@ public NetworkLocation get(HostAddress host) rackLocalSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(dataHost2))); } assignments = nodeSelector.computeAssignments(rackLocalSplits.build(), ImmutableList.copyOf(taskMap.values())).getAssignments(); - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { RemoteTask remoteTask = taskMap.get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -227,7 +226,7 @@ public NetworkLocation get(HostAddress host) MILLISECONDS.sleep(10); } assignments = nodeSelector.computeAssignments(unassigned, ImmutableList.copyOf(taskMap.values())).getAssignments(); - for (Node node : assignments.keySet()) { + for (InternalNode node : assignments.keySet()) { RemoteTask remoteTask = taskMap.get(node); remoteTask.addSplits(ImmutableMultimap.builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) @@ -268,7 +267,7 @@ public void testScheduleRemote() { Set splits = new HashSet<>(); splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())); - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); assertEquals(assignments.size(), 1); } @@ -280,9 +279,9 @@ public void testBasicAssignment() for (int i = 0; i < 3; i++) { splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())); } - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); assertEquals(assignments.entries().size(), 3); - for (Node node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { + for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { assertTrue(assignments.keySet().contains(node)); } } @@ -292,7 +291,7 @@ public void testMaxSplitsPerNode() { TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); - Node newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); @@ -314,7 +313,7 @@ public void testMaxSplitsPerNode() for (int i = 0; i < 5; i++) { splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); // no split should be assigned to the newNode, as it already has maxNodeSplits assigned to it assertFalse(assignments.keySet().contains(newNode)); @@ -330,7 +329,7 @@ public void testMaxSplitsPerNodePerTask() { TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); - Node newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode); ImmutableList.Builder initialSplits = ImmutableList.builder(); @@ -340,7 +339,7 @@ public void testMaxSplitsPerNodePerTask() List tasks = new ArrayList<>(); MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); - for (Node node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { + for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { // Max out number of splits on node TaskId taskId = new TaskId("test", 1, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); @@ -359,7 +358,7 @@ public void testMaxSplitsPerNodePerTask() for (int i = 0; i < 5; i++) { splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); } - Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); + Multimap assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments(); // no split should be assigned to the newNode, as it already has // maxSplitsPerNode + maxSplitsPerNodePerTask assigned to it @@ -377,7 +376,7 @@ public void testTaskCompletion() throws Exception { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); - Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); + InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); TaskId taskId = new TaskId("test", 1, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask( taskId, @@ -398,7 +397,7 @@ public void testTaskCompletion() public void testSplitCount() { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); - Node chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); + InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); TaskId taskId1 = new TaskId("test", 1, 1); RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java index 355e45d8681f8..48ee3b2fe3879 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java @@ -20,7 +20,6 @@ import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.StageExecutionDescriptor; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.Partitioning; @@ -126,7 +125,7 @@ private void testFinalStageInfoInternal() if (Thread.interrupted()) { return; } - Node node = new InternalNode( + InternalNode node = new InternalNode( "source" + i, URI.create("http://10.0.0." + (i / 10_000) + ":" + (i % 10_000)), NodeVersion.UNKNOWN, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java index 7a6387a454ef4..0c4745f3216c7 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java @@ -21,9 +21,9 @@ import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.memory.NodeMemoryConfig; import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.ExchangeClient; import com.facebook.presto.operator.ExchangeClientSupplier; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spiller.LocalSpillManager; import com.facebook.presto.spiller.NodeSpillConfig; @@ -310,13 +310,13 @@ public URI createLocalTaskLocation(TaskId taskId) } @Override - public URI createTaskLocation(Node node, TaskId taskId) + public URI createTaskLocation(InternalNode node, TaskId taskId) { return URI.create("http://fake.invalid/task/" + node.getNodeIdentifier() + "/" + taskId); } @Override - public URI createMemoryInfoLocation(Node node) + public URI createMemoryInfoLocation(InternalNode node) { return URI.create("http://fake.invalid/" + node.getNodeIdentifier() + "/memory"); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java index 2c5e96c1d79c6..8fcc8a69253ec 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java @@ -19,7 +19,6 @@ import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.TaskId; import com.facebook.presto.metadata.InternalNode; -import com.facebook.presto.spi.Node; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -91,7 +90,7 @@ public void testMultipleNodes() assertEquals(result.getNewTasks().stream().map(RemoteTask::getNodeId).collect(toImmutableSet()).size(), 5); } - private static List generateRandomNodes(int count) + private static List generateRandomNodes(int count) { return IntStream.range(0, count) .mapToObj(i -> new InternalNode("other " + i, URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)) diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index 635784b11707e..160be40f3d2ac 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -27,14 +27,13 @@ import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.metadata.InMemoryNodeManager; -import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.InternalNode; +import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.TableHandle; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.FixedSplitSource; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; @@ -355,7 +354,7 @@ public void testBalancedSplitAssignment() } // Add new node - Node additionalNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); + InternalNode additionalNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, additionalNode); // Schedule 5 splits in another query. Since the new node does not have any splits, all 5 splits are assigned to the new node diff --git a/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java b/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java index eb31eda96ad1b..e3f3650e89669 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/LowMemoryKillerTestingUtils.java @@ -16,7 +16,6 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.metadata.InternalNode; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spi.memory.MemoryPoolInfo; @@ -39,7 +38,7 @@ private LowMemoryKillerTestingUtils() {} static List toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map> queries) { - Map nodeReservations = new HashMap<>(); + Map nodeReservations = new HashMap<>(); for (Map.Entry> entry : queries.entrySet()) { QueryId queryId = new QueryId(entry.getKey()); @@ -61,7 +60,7 @@ static List toNodeMemoryInfoList(long maxReservedPoolBytes, long max } ImmutableList.Builder result = ImmutableList.builder(); - for (Map.Entry entry : nodeReservations.entrySet()) { + for (Map.Entry entry : nodeReservations.entrySet()) { NodeReservation nodeReservation = entry.getValue(); ImmutableMap.Builder pools = ImmutableMap.builder(); if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) { diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java index e4f43687a0280..afc93ff1fc784 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java @@ -16,7 +16,6 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.server.InternalCommunicationConfig; -import com.facebook.presto.spi.Node; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -56,8 +55,8 @@ public class TestDiscoveryNodeManager private final NodeInfo nodeInfo = new NodeInfo("test"); private final InternalCommunicationConfig internalCommunicationConfig = new InternalCommunicationConfig(); private NodeVersion expectedVersion; - private Set activeNodes; - private Set inactiveNodes; + private Set activeNodes; + private Set inactiveNodes; private InternalNode coordinator; private InternalNode currentNode; private final PrestoNodeServiceSelector selector = new PrestoNodeServiceSelector(); @@ -91,22 +90,22 @@ public void testGetAllNodes() try { AllNodes allNodes = manager.getAllNodes(); - Set activeNodes = allNodes.getActiveNodes(); + Set activeNodes = allNodes.getActiveNodes(); assertEqualsIgnoreOrder(activeNodes, this.activeNodes); - for (Node actual : activeNodes) { - for (Node expected : this.activeNodes) { + for (InternalNode actual : activeNodes) { + for (InternalNode expected : this.activeNodes) { assertNotSame(actual, expected); } } assertEqualsIgnoreOrder(activeNodes, manager.getNodes(ACTIVE)); - Set inactiveNodes = allNodes.getInactiveNodes(); + Set inactiveNodes = allNodes.getInactiveNodes(); assertEqualsIgnoreOrder(inactiveNodes, this.inactiveNodes); - for (Node actual : inactiveNodes) { - for (Node expected : this.inactiveNodes) { + for (InternalNode actual : inactiveNodes) { + for (InternalNode expected : this.inactiveNodes) { assertNotSame(actual, expected); } } @@ -188,14 +187,14 @@ public static class PrestoNodeServiceSelector @GuardedBy("this") private List descriptors = ImmutableList.of(); - private synchronized void announceNodes(Set activeNodes, Set inactiveNodes) + private synchronized void announceNodes(Set activeNodes, Set inactiveNodes) { ImmutableList.Builder descriptors = ImmutableList.builder(); - for (Node node : Iterables.concat(activeNodes, inactiveNodes)) { + for (InternalNode node : Iterables.concat(activeNodes, inactiveNodes)) { descriptors.add(serviceDescriptor("presto") .setNodeId(node.getNodeIdentifier()) .addProperty("http", node.getHttpUri().toString()) - .addProperty("node_version", ((InternalNode) node).getNodeVersion().toString()) + .addProperty("node_version", node.getNodeVersion().toString()) .addProperty("coordinator", String.valueOf(node.isCoordinator())) .build()); } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index dc80176ae95a6..fff505dd730df 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -22,12 +22,12 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.AllNodes; import com.facebook.presto.metadata.Catalog; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.testing.TestingPrestoServer; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.QueryId; import com.facebook.presto.split.PageSourceManager; @@ -332,7 +332,7 @@ private boolean isConnectionVisibleToAllNodes(ConnectorId connectorId) { for (TestingPrestoServer server : servers) { server.refreshNodes(); - Set activeNodesWithConnector = server.getActiveNodesWithConnector(connectorId); + Set activeNodesWithConnector = server.getActiveNodesWithConnector(connectorId); if (activeNodesWithConnector.size() != servers.size()) { return false; } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java index efea4c2cadb01..862d9f8c91192 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java @@ -17,11 +17,11 @@ import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.cost.StatsCalculator; import com.facebook.presto.metadata.AllNodes; +import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.server.testing.TestingPrestoServer; -import com.facebook.presto.spi.Node; import com.facebook.presto.spi.Plugin; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; @@ -190,7 +190,7 @@ public void refreshNodes() private void refreshNodes(ConnectorId connectorId) { - Set activeNodesWithConnector; + Set activeNodesWithConnector; do { try { From 2e199b0e64b346c6641b67b70ee442f1b0617c76 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Tue, 2 Apr 2019 15:54:22 -0700 Subject: [PATCH 16/66] Deprecate Node.getHttpUri and reduce usages of Node.getHostAndPort Extracted-From: https://github.com/prestosql/presto --- .../presto/hive/HiveWriterFactory.java | 2 +- .../connector/system/NodeSystemTable.java | 2 +- .../execution/scheduler/NodeScheduler.java | 2 +- .../presto/memory/ClusterMemoryManager.java | 2 +- .../presto/metadata/DiscoveryNodeManager.java | 4 ++-- .../presto/metadata/InternalNode.java | 24 ++++++++++++++----- .../presto/server/WorkerResource.java | 2 +- .../remotetask/HttpLocationFactory.java | 4 ++-- .../metadata/TestDiscoveryNodeManager.java | 2 +- .../java/com/facebook/presto/spi/Node.java | 6 +++++ 10 files changed, 34 insertions(+), 16 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java index 3eb6c5f6cd8c9..c503af4049f94 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java @@ -494,7 +494,7 @@ else if (insertExistingPartitionsBehavior == InsertExistingPartitionsBehavior.ER outputStorageFormat.getOutputFormat(), writerImplementation, nodeManager.getCurrentNode().getVersion(), - nodeManager.getCurrentNode().getHttpUri().getHost(), + nodeManager.getCurrentNode().getHost(), session.getIdentity().getPrincipal().map(Principal::getName).orElse(null), nodeManager.getEnvironment(), sessionProperties, diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java index ef90e994cb57e..68327e6d2e334 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/NodeSystemTable.java @@ -88,7 +88,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect private void addRows(Builder table, Set nodes, NodeState state) { for (InternalNode node : nodes) { - table.addRow(node.getNodeIdentifier(), node.getHttpUri().toString(), getNodeVersion(node), isCoordinator(node), state.toString().toLowerCase(Locale.ENGLISH)); + table.addRow(node.getNodeIdentifier(), node.getInternalUri().toString(), getNodeVersion(node), isCoordinator(node), state.toString().toLowerCase(Locale.ENGLISH)); } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java index e325a6b0949a9..101b873af7c99 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java @@ -151,7 +151,7 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) try { byHostAndPort.put(node.getHostAndPort(), node); - InetAddress host = InetAddress.getByName(node.getHttpUri().getHost()); + InetAddress host = InetAddress.getByName(node.getInternalUri().getHost()); byHost.put(host, node); } catch (UnknownHostException e) { diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java index b9d9749d0c676..9ff89fa220a69 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java @@ -551,7 +551,7 @@ public synchronized Map> getWorkerMemoryInfo() Map> memoryInfo = new HashMap<>(); for (Entry entry : nodes.entrySet()) { // workerId is of the form "node_identifier [node_host]" - String workerId = entry.getKey() + " [" + entry.getValue().getNode().getHostAndPort().getHostText() + "]"; + String workerId = entry.getKey() + " [" + entry.getValue().getNode().getHost() + "]"; memoryInfo.put(workerId, entry.getValue().getInfo()); } return memoryInfo; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index 6bd45c3c0fde7..7e1b60f959655 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -175,7 +175,7 @@ private void pollWorkers() // Add new nodes for (InternalNode node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), - new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); + new RemoteNodeState(httpClient, uriBuilderFrom(node.getInternalUri()).appendPath("/v1/info/state").build())); } // Schedule refresh @@ -254,7 +254,7 @@ private synchronized void refreshNodesInternal() // log node that are no longer active (but not shutting down) SetView missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); for (InternalNode missingNode : missingNodes) { - log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHostAndPort()); + log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHost()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java index 65b6ca5dc14de..b869e4375b7bb 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/InternalNode.java @@ -31,15 +31,15 @@ public class InternalNode implements Node { private final String nodeIdentifier; - private final URI httpUri; + private final URI internalUri; private final NodeVersion nodeVersion; private final boolean coordinator; - public InternalNode(String nodeIdentifier, URI httpUri, NodeVersion nodeVersion, boolean coordinator) + public InternalNode(String nodeIdentifier, URI internalUri, NodeVersion nodeVersion, boolean coordinator) { nodeIdentifier = emptyToNull(nullToEmpty(nodeIdentifier).trim()); this.nodeIdentifier = requireNonNull(nodeIdentifier, "nodeIdentifier is null or empty"); - this.httpUri = requireNonNull(httpUri, "httpUri is null"); + this.internalUri = requireNonNull(internalUri, "internalUri is null"); this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null"); this.coordinator = coordinator; } @@ -51,15 +51,27 @@ public String getNodeIdentifier() } @Override + public String getHost() + { + return internalUri.getHost(); + } + + @Override + @Deprecated public URI getHttpUri() { - return httpUri; + return getInternalUri(); + } + + public URI getInternalUri() + { + return internalUri; } @Override public HostAddress getHostAndPort() { - return HostAddress.fromUri(httpUri); + return HostAddress.fromUri(internalUri); } @Override @@ -103,7 +115,7 @@ public String toString() { return toStringHelper(this) .add("nodeIdentifier", nodeIdentifier) - .add("httpUri", httpUri) + .add("internalUri", internalUri) .add("nodeVersion", nodeVersion) .toString(); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java index 7672f84ff57b2..b1a575a37e479 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java @@ -75,7 +75,7 @@ private Response proxyJsonResponse(String nodeId, String workerPath) .orElseThrow(() -> new WebApplicationException(NOT_FOUND)); Request request = prepareGet() - .setUri(uriBuilderFrom(node.getHttpUri()) + .setUri(uriBuilderFrom(node.getInternalUri()) .appendPath(workerPath) .build()) .build(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java index 6c88bed17f740..ee961e63c948f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java @@ -78,7 +78,7 @@ public URI createTaskLocation(InternalNode node, TaskId taskId) { requireNonNull(node, "node is null"); requireNonNull(taskId, "taskId is null"); - return uriBuilderFrom(node.getHttpUri()) + return uriBuilderFrom(node.getInternalUri()) .appendPath("/v1/task") .appendPath(taskId.toString()) .build(); @@ -88,7 +88,7 @@ public URI createTaskLocation(InternalNode node, TaskId taskId) public URI createMemoryInfoLocation(InternalNode node) { requireNonNull(node, "node is null"); - return uriBuilderFrom(node.getHttpUri()) + return uriBuilderFrom(node.getInternalUri()) .appendPath("/v1/memory").build(); } } diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java index afc93ff1fc784..785170957bad2 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java @@ -193,7 +193,7 @@ private synchronized void announceNodes(Set activeNodes, Set Date: Thu, 11 Apr 2019 20:25:12 -0700 Subject: [PATCH 17/66] Auto-resolve EXCEEDED_TIME_LIMIT regardless of CPU time increase --- .../ExceededTimeLimitFailureResolver.java | 3 +-- .../TestExceededTimeLimitFailureResolver.java | 15 --------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java index b93c87ca8189b..ee9c0a2cbf896 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java @@ -33,8 +33,7 @@ public ExceededTimeLimitFailureResolver() @Override public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats) { - if (errorCode == EXCEEDED_TIME_LIMIT && - controlQueryStats.getCpuTimeMillis() > testQueryStats.getCpuTimeMillis()) { + if (errorCode == EXCEEDED_TIME_LIMIT) { return Optional.of("Auto Resolved: Test cluster has less computing resource"); } return Optional.empty(); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java index a7449b9cfbfcc..e16c710af0edd 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java @@ -22,7 +22,6 @@ import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; public class TestExceededTimeLimitFailureResolver extends AbstractTestPrestoQueryFailureResolver @@ -32,20 +31,6 @@ public TestExceededTimeLimitFailureResolver() super(new ExceededTimeLimitFailureResolver()); } - @Test - public void testHigherCpuTime() - { - assertFalse(getFailureResolver().resolve( - CONTROL_QUERY_STATS, - QueryException.forPresto( - new RuntimeException(), - Optional.of(EXCEEDED_TIME_LIMIT), - false, - Optional.of(createQueryStats(CONTROL_CPU_TIME_MILLIS * 2, CONTROL_PEAK_MEMORY_BYTES)), - forMain(TEST))) - .isPresent()); - } - @Test public void testResolved() { From a0f64ed14cdea107aeb8c7247c793a49f4034ae2 Mon Sep 17 00:00:00 2001 From: jh_lee Date: Fri, 19 Apr 2019 03:00:16 +0900 Subject: [PATCH 18/66] Add support for readPreferenceTags in mongodb Add support for readPreferenceTags that lead the mongodb connector to read a specific sharded cluster to configuration properties. The properties are split tag sets as a character '&' and specified a tag set as a comma-separated list of colon-separated key-value paris. For example, mongodb.read-preference-tags=dc:east,use:reporting&use:reporting --- .../presto/mongodb/MongoClientConfig.java | 37 +++++++++++++++++++ .../presto/mongodb/MongoClientModule.java | 8 +++- .../presto/mongodb/ReadPreferenceType.java | 14 +++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java index 6195a7f9ff931..9d16b73ebe162 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java @@ -17,6 +17,8 @@ import com.google.common.collect.ImmutableList; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; +import com.mongodb.Tag; +import com.mongodb.TagSet; import io.airlift.configuration.Config; import io.airlift.configuration.DefunctConfig; @@ -36,6 +38,8 @@ public class MongoClientConfig private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); private static final Splitter PORT_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); private static final Splitter USER_SPLITTER = Splitter.onPattern("[:@]").trimResults().omitEmptyStrings(); + private static final Splitter TAGSET_SPLITTER = Splitter.on('&').trimResults().omitEmptyStrings(); + private static final Splitter TAG_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); private String schemaCollection = "_schema"; private List seeds = ImmutableList.of(); @@ -53,6 +57,7 @@ public class MongoClientConfig private int cursorBatchSize; // use driver default private ReadPreferenceType readPreference = ReadPreferenceType.PRIMARY; + private List readPreferenceTagSets = ImmutableList.of(); private WriteConcernType writeConcern = WriteConcernType.ACKNOWLEDGED; private String requiredReplicaSetName; private String implicitRowFieldPrefix = "_pos"; @@ -226,6 +231,38 @@ public MongoClientConfig setReadPreference(ReadPreferenceType readPreference) return this; } + public List getReadPreferenceTags() + { + return readPreferenceTagSets; + } + + @Config("mongodb.read-preference-tags") + public MongoClientConfig setReadPreferenceTags(String readPreferenceTags) + { + this.readPreferenceTagSets = buildTagSets(TAGSET_SPLITTER.split(readPreferenceTags)); + return this; + } + + private List buildTagSets(Iterable tagSets) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String tagSet : tagSets) { + builder.add(new TagSet(buildTags(SPLITTER.split(tagSet)))); + } + return builder.build(); + } + + private List buildTags(Iterable tags) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String tag : tags) { + List values = TAG_SPLITTER.splitToList(tag); + checkArgument(values.size() == 2, "Invalid Tag format. Requires tagName:tagValue"); + builder.add(new Tag(values.get(0), values.get(1))); + } + return builder.build(); + } + public WriteConcernType getWriteConcern() { return writeConcern; diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java index b0f54dd5eae9a..58c7cae22da45 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java @@ -55,13 +55,19 @@ public static MongoSession createMongoSession(TypeManager typeManager, MongoClie .sslEnabled(config.getSslEnabled()) .maxWaitTime(config.getMaxWaitTime()) .minConnectionsPerHost(config.getMinConnectionsPerHost()) - .readPreference(config.getReadPreference().getReadPreference()) .writeConcern(config.getWriteConcern().getWriteConcern()); if (config.getRequiredReplicaSetName() != null) { options.requiredReplicaSetName(config.getRequiredReplicaSetName()); } + if (config.getReadPreferenceTags().isEmpty()) { + options.readPreference(config.getReadPreference().getReadPreference()); + } + else { + options.readPreference(config.getReadPreference().getReadPreferenceWithTags(config.getReadPreferenceTags())); + } + MongoClient client = new MongoClient(config.getSeeds(), config.getCredentials(), options.build()); return new MongoSession( diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java index 0c779f6ff45f9..f1fbec0a6bd92 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/ReadPreferenceType.java @@ -13,8 +13,13 @@ */ package com.facebook.presto.mongodb; +import com.facebook.presto.spi.PrestoException; import com.mongodb.ReadPreference; +import com.mongodb.TagSet; +import java.util.List; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static java.util.Objects.requireNonNull; public enum ReadPreferenceType @@ -36,4 +41,13 @@ public ReadPreference getReadPreference() { return readPreference; } + + public ReadPreference getReadPreferenceWithTags(List tagSets) + { + if (PRIMARY.equals(this)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Primary read preference can not specify tag sets"); + } + + return ReadPreference.valueOf(readPreference.getName(), tagSets); + } } From 91599cfd76fe78a1fc429ba1499883df7019f9a3 Mon Sep 17 00:00:00 2001 From: James Sun Date: Sat, 6 Apr 2019 23:37:27 -0700 Subject: [PATCH 19/66] Remove Raptor OrcStorageManager::appendRow --- .../presto/raptor/storage/OrcFileWriter.java | 2 +- .../raptor/storage/OrcStorageManager.java | 7 ----- .../raptor/storage/StoragePageSink.java | 2 -- .../storage/organization/ShardCompactor.java | 27 +++++++++---------- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java index 739aaf2656133..891a542d31992 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java @@ -147,7 +147,7 @@ public void appendPages(List inputPages, int[] pageIndexes, int[] position } } - public void appendRow(Row row) + private void appendRow(Row row) { List columns = row.getColumns(); checkArgument(columns.size() == columnTypes.size()); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index 91f6566aebe12..0e4a75eb7f394 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -589,13 +589,6 @@ public void appendPages(List inputPages, int[] pageIndexes, int[] position writer.appendPages(inputPages, pageIndexes, positionIndexes); } - @Override - public void appendRow(Row row) - { - createWriterIfNecessary(); - writer.appendRow(row); - } - @Override public boolean isFull() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java index 88232ff0f135f..4787e8ffced4d 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StoragePageSink.java @@ -25,8 +25,6 @@ public interface StoragePageSink void appendPages(List pages, int[] pageIndexes, int[] positionIndexes); - void appendRow(Row row); - boolean isFull(); void flush(); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java index 256cb42f66e74..6c99abfcd0dbd 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java @@ -16,7 +16,6 @@ import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.storage.ReaderAttributes; -import com.facebook.presto.raptor.storage.Row; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.storage.StoragePageSink; import com.facebook.presto.spi.ConnectorPageSource; @@ -44,7 +43,6 @@ import java.util.Set; import java.util.UUID; -import static com.facebook.presto.raptor.storage.Row.extractRow; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.airlift.units.Duration.nanosSince; @@ -128,23 +126,23 @@ public List compactSorted(long transactionId, OptionalInt bucketNumbe .map(columnIds::indexOf) .collect(toList()); - Queue rowSources = new PriorityQueue<>(); + Queue rowSources = new PriorityQueue<>(); StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); try { for (UUID uuid : uuids) { ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes); - SortedRowSource rowSource = new SortedRowSource(pageSource, columnTypes, sortIndexes, sortOrders); + SortedPageSource rowSource = new SortedPageSource(pageSource, columnTypes, sortIndexes, sortOrders); rowSources.add(rowSource); } while (!rowSources.isEmpty()) { - SortedRowSource rowSource = rowSources.poll(); + SortedPageSource rowSource = rowSources.poll(); if (!rowSource.hasNext()) { // rowSource is empty, close it rowSource.close(); continue; } - outputPageSink.appendRow(rowSource.next()); + outputPageSink.appendPages(ImmutableList.of(rowSource.next())); if (outputPageSink.isFull()) { outputPageSink.flush(); @@ -164,12 +162,12 @@ public List compactSorted(long transactionId, OptionalInt bucketNumbe throw e; } finally { - rowSources.forEach(SortedRowSource::closeQuietly); + rowSources.forEach(SortedPageSource::closeQuietly); } } - private static class SortedRowSource - implements Iterator, Comparable, Closeable + private static class SortedPageSource + implements Iterator, Comparable, Closeable { private final ConnectorPageSource pageSource; private final List columnTypes; @@ -179,7 +177,7 @@ private static class SortedRowSource private Page currentPage; private int currentPosition; - public SortedRowSource(ConnectorPageSource pageSource, List columnTypes, List sortIndexes, List sortOrders) + public SortedPageSource(ConnectorPageSource pageSource, List columnTypes, List sortIndexes, List sortOrders) { this.pageSource = requireNonNull(pageSource, "pageSource is null"); this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); @@ -219,19 +217,20 @@ private static Page getNextPage(ConnectorPageSource pageSource) } @Override - public Row next() + public Page next() { if (!hasNext()) { throw new NoSuchElementException(); } - Row row = extractRow(currentPage, currentPosition, columnTypes); + int[] mask = {currentPosition}; + Page page = currentPage.getPositions(mask, 0, 1); currentPosition++; - return row; + return page; } @Override - public int compareTo(SortedRowSource other) + public int compareTo(SortedPageSource other) { if (!hasNext()) { return 1; From 41ee5191787cb14ecadef6818500ff2b1a0d8a40 Mon Sep 17 00:00:00 2001 From: James Sun Date: Sat, 6 Apr 2019 23:37:28 -0700 Subject: [PATCH 20/66] Rename Raptor OrcFileWriter to OrcRecordWriter Previous OrcFileWriter is not columnarized. Rename it to OrcRecordWriter. Abstract out FileWriter for preparation of introducing new Orc writer. --- ...rcFileWriter.java => OrcRecordWriter.java} | 59 +++++++++---------- .../raptor/storage/OrcStorageManager.java | 22 +++++-- .../raptor/storage/TestOrcFileRewriter.java | 10 ++-- .../raptor/storage/TestShardWriter.java | 4 +- 4 files changed, 53 insertions(+), 42 deletions(-) rename presto-raptor/src/main/java/com/facebook/presto/raptor/storage/{OrcFileWriter.java => OrcRecordWriter.java} (95%) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java similarity index 95% rename from presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java rename to presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java index 891a542d31992..d690c8e596521 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; @@ -82,8 +81,8 @@ import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; -public class OrcFileWriter - implements Closeable +public class OrcRecordWriter + implements FileWriter { private static final Configuration CONFIGURATION = new Configuration(); private static final Constructor WRITER_CONSTRUCTOR = getOrcWriterConstructor(); @@ -101,13 +100,13 @@ public class OrcFileWriter private long rowCount; private long uncompressedSize; - public OrcFileWriter(List columnIds, List columnTypes, File target) + public OrcRecordWriter(List columnIds, List columnTypes, File target) { this(columnIds, columnTypes, target, true); } @VisibleForTesting - OrcFileWriter(List columnIds, List columnTypes, File target, boolean writeMetadata) + OrcRecordWriter(List columnIds, List columnTypes, File target, boolean writeMetadata) { this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); @@ -129,6 +128,7 @@ public OrcFileWriter(List columnIds, List columnTypes, File target) orcRow = tableInspector.create(); } + @Override public void appendPages(List pages) { for (Page page : pages) { @@ -138,6 +138,7 @@ public void appendPages(List pages) } } + @Override public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) { checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); @@ -147,49 +148,47 @@ public void appendPages(List inputPages, int[] pageIndexes, int[] position } } - private void appendRow(Row row) - { - List columns = row.getColumns(); - checkArgument(columns.size() == columnTypes.size()); - for (int channel = 0; channel < columns.size(); channel++) { - tableInspector.setStructFieldData(orcRow, structFields.get(channel), columns.get(channel)); - } - try { - recordWriter.write(serializer.serialize(orcRow, tableInspector)); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to write record", e); - } - rowCount++; - uncompressedSize += row.getSizeInBytes(); - } - @Override public void close() + throws IOException { if (closed) { return; } closed = true; - try { - recordWriter.close(false); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to close writer", e); - } + recordWriter.close(false); } + @Override public long getRowCount() { return rowCount; } + @Override public long getUncompressedSize() { return uncompressedSize; } + private void appendRow(Row row) + { + List columns = row.getColumns(); + checkArgument(columns.size() == columnTypes.size()); + for (int channel = 0; channel < columns.size(); channel++) { + tableInspector.setStructFieldData(orcRow, structFields.get(channel), columns.get(channel)); + } + try { + recordWriter.write(serializer.serialize(orcRow, tableInspector)); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to write record", e); + } + rowCount++; + uncompressedSize += row.getSizeInBytes(); + } + private static OrcSerde createSerializer(Properties properties) { OrcSerde serde = new OrcSerde(); @@ -258,7 +257,7 @@ private static List getJavaObjectInspectors(List t return types.stream() .map(StorageType::getHiveTypeName) .map(TypeInfoUtils::getTypeInfoFromTypeString) - .map(OrcFileWriter::getJavaObjectInspector) + .map(OrcRecordWriter::getJavaObjectInspector) .collect(toList()); } @@ -288,7 +287,7 @@ private static boolean isUnique(Collection items) private static List toStorageTypes(List columnTypes) { - return columnTypes.stream().map(OrcFileWriter::toStorageType).collect(toList()); + return columnTypes.stream().map(OrcRecordWriter::toStorageType).collect(toList()); } private static StorageType toStorageType(Type type) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index 0e4a75eb7f394..4504fdd568647 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -564,7 +564,7 @@ private class OrcStoragePageSink private final List> futures = new ArrayList<>(); private boolean committed; - private OrcFileWriter writer; + private FileWriter writer; private UUID shardUuid; public OrcStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) @@ -602,7 +602,12 @@ public boolean isFull() public void flush() { if (writer != null) { - writer.close(); + try { + writer.close(); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to close writer", e); + } shardRecorder.recordCreatedShard(transactionId, shardUuid); @@ -642,8 +647,15 @@ public void rollback() { try { if (writer != null) { - writer.close(); - writer = null; + try { + writer.close(); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to close writer", e); + } + finally { + writer = null; + } } } finally { @@ -670,7 +682,7 @@ private void createWriterIfNecessary() File stagingFile = storageService.getStagingFile(shardUuid); storageService.createParents(stagingFile); stagingFiles.add(stagingFile); - writer = new OrcFileWriter(columnIds, columnTypes, stagingFile); + writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile); } } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java index af99c73b61b04..4295588cce08e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java @@ -103,7 +103,7 @@ public void testRewrite() List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")) .row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")) @@ -272,7 +272,7 @@ public void testRewriteWithoutMetadata() List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file, false)) { + try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file, false)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello") .row(777L, "sky") @@ -347,7 +347,7 @@ public void testRewriteAllRowsDeleted() List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } @@ -371,7 +371,7 @@ public void testRewriteNoRowsDeleted() List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } @@ -393,7 +393,7 @@ public void testUncompressedSize() List columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY); File file = new File(temporary, randomUUID().toString()); - try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { List pages = rowPagesBuilder(columnTypes) .row(true, 123L, 98.7, "hello", utf8Slice("abc")) .row(false, 456L, 65.4, "world", utf8Slice("xyz")) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java index db4ca32adb0a8..88bf81bedc6dd 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java @@ -106,7 +106,7 @@ public void testWriter() .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader()); - OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { + FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder.build()); } @@ -207,7 +207,7 @@ public void testWriterZeroRows() File file = new File(directory, System.nanoTime() + ".orc"); - try (OrcFileWriter ignored = new OrcFileWriter(columnIds, columnTypes, file)) { + try (OrcRecordWriter ignored = new OrcRecordWriter(columnIds, columnTypes, file)) { // no rows } From 7bbd393f42312734270576b8480a9d3e6da8044c Mon Sep 17 00:00:00 2001 From: James Sun Date: Sat, 6 Apr 2019 23:37:30 -0700 Subject: [PATCH 21/66] Use single source of truth for ORC timezone --- .../facebook/presto/raptor/storage/OrcStorageManager.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index 4504fdd568647..b7a6dd1803fb0 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -127,6 +127,12 @@ public class OrcStorageManager implements StorageManager { + // Raptor does not store time-related data types as they are in ORC. + // They will be casted to BIGINT or INTEGER to avoid timezone conversion. + // This is due to the historical reason of using the legacy ORC read/writer that does not support timestamp types. + // In order to be consistent, we still enforce the conversion. + // The following DEFAULT_STORAGE_TIMEZONE is not used by the optimized ORC read/writer given we never read/write timestamp types. + public static final DateTimeZone DEFAULT_STORAGE_TIMEZONE = UTC; private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); private static final long MAX_ROWS = 1_000_000_000; @@ -259,7 +265,7 @@ public ConnectorPageSource getPageSource( OrcPredicate predicate = getPredicate(effectivePredicate, indexMap); - OrcRecordReader recordReader = reader.createRecordReader(includedColumns.build(), predicate, UTC, systemMemoryUsage, INITIAL_BATCH_SIZE); + OrcRecordReader recordReader = reader.createRecordReader(includedColumns.build(), predicate, DEFAULT_STORAGE_TIMEZONE, systemMemoryUsage, INITIAL_BATCH_SIZE); Optional shardRewriter = Optional.empty(); if (transactionId.isPresent()) { From 8ab71a2f8f24a4dd8966af5cc31e4a32975fe56c Mon Sep 17 00:00:00 2001 From: James Sun Date: Sat, 6 Apr 2019 23:37:31 -0700 Subject: [PATCH 22/66] Add optimized ORC writer to Raptor FileWriter --- .../presto/raptor/RaptorErrorCode.java | 3 +- .../presto/raptor/metadata/ColumnStats.java | 23 +++ .../raptor/storage/OrcFileRewriter.java | 27 +++ .../presto/raptor/storage/OrcFileWriter.java | 177 ++++++++++++++++++ .../raptor/storage/OrcRecordWriter.java | 4 + .../raptor/storage/OrcStorageManager.java | 27 ++- .../presto/raptor/storage/ShardStats.java | 2 + .../raptor/storage/StorageManagerConfig.java | 19 ++ .../raptor/storage/StorageTypeConverter.java | 99 ++++++++++ .../presto/raptor/TestRaptorConnector.java | 2 +- .../presto/raptor/storage/OrcTestingUtil.java | 10 + .../raptor/storage/TestOrcFileRewriter.java | 51 +++-- .../raptor/storage/TestOrcStorageManager.java | 96 +++++++--- .../raptor/storage/TestShardWriter.java | 90 ++++++++- .../storage/TestStorageManagerConfig.java | 5 + .../organization/TestShardCompactor.java | 53 +++--- 16 files changed, 606 insertions(+), 82 deletions(-) create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java index 5a867b419c3cf..09da294fac836 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java @@ -36,7 +36,8 @@ public enum RaptorErrorCode RAPTOR_CORRUPT_METADATA(11, EXTERNAL), RAPTOR_LOCAL_DISK_FULL(12, EXTERNAL), RAPTOR_BACKUP_CORRUPTION(13, EXTERNAL), - RAPTOR_NOT_ENOUGH_NODES(14, EXTERNAL); + RAPTOR_NOT_ENOUGH_NODES(14, EXTERNAL), + RAPTOR_WRITER_DATA_ERROR(15, EXTERNAL); private final ErrorCode errorCode; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java index 85154159f6eb0..544e405d02b91 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ColumnStats.java @@ -18,6 +18,8 @@ import javax.annotation.Nullable; +import java.util.Objects; + import static com.google.common.base.MoreObjects.toStringHelper; public class ColumnStats @@ -67,4 +69,25 @@ public String toString() .omitNullValues() .toString(); } + + @Override + public boolean equals(Object other) + { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + ColumnStats that = (ColumnStats) other; + return Objects.equals(columnId, that.columnId) && + Objects.equals(min, that.min) && + Objects.equals(max, that.max); + } + + @Override + public int hashCode() + { + return Objects.hash(columnId, min, max); + } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java index 7ac52c6b57863..9b7ed952b0f1d 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java @@ -27,10 +27,16 @@ import org.apache.hadoop.hive.ql.io.orc.Reader; import org.apache.hadoop.hive.ql.io.orc.RecordReader; import org.apache.hadoop.hive.ql.io.orc.Writer; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -45,7 +51,10 @@ import static com.facebook.presto.raptor.util.Closer.closer; import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; +import static io.airlift.slice.SizeOf.SIZE_OF_FLOAT; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; import static io.airlift.slice.SizeOf.SIZE_OF_LONG; +import static io.airlift.slice.SizeOf.SIZE_OF_SHORT; import static io.airlift.units.Duration.nanosSince; import static java.lang.Math.toIntExact; import static org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader; @@ -166,6 +175,24 @@ private static int uncompressedSize(Object object) if (object instanceof BytesWritable) { return ((BytesWritable) object).getLength(); } + if (object instanceof ByteWritable) { + return SIZE_OF_BYTE; + } + if (object instanceof HiveVarcharWritable) { + return ((HiveVarcharWritable) object).getTextValue().getLength(); + } + if (object instanceof DateWritable) { + return SIZE_OF_INT; + } + if (object instanceof IntWritable) { + return SIZE_OF_INT; + } + if (object instanceof ShortWritable) { + return SIZE_OF_SHORT; + } + if (object instanceof FloatWritable) { + return SIZE_OF_FLOAT; + } if (object instanceof List) { int size = 0; for (Object element : (Iterable) object) { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java new file mode 100644 index 0000000000000..e5dcae8cd7fa0 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java @@ -0,0 +1,177 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.orc.OrcWriter; +import com.facebook.presto.orc.OrcWriterOptions; +import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import org.apache.hadoop.fs.FileSystem; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_WRITER_DATA_ERROR; +import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public class OrcFileWriter + implements FileWriter +{ + private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); + private static final OrcWriterOptions DEFAULT_OPTION = new OrcWriterOptions(); + + private final OrcWriter orcWriter; + + private boolean closed; + private long rowCount; + private long uncompressedSize; + + public OrcFileWriter(List columnIds, List columnTypes, File target, boolean validate, OrcWriterStats stats, TypeManager typeManager) + { + this(columnIds, columnTypes, target, true, validate, stats, typeManager); + } + + @VisibleForTesting + OrcFileWriter( + List columnIds, + List columnTypes, + File target, + boolean writeMetadata, + boolean validate, + OrcWriterStats stats, + TypeManager typeManager) + { + checkArgument(requireNonNull(columnIds, "columnIds is null").size() == requireNonNull(columnTypes, "columnTypes is null").size(), "ids and types mismatch"); + checkArgument(isUnique(columnIds), "ids must be unique"); + + StorageTypeConverter converter = new StorageTypeConverter(typeManager); + List storageTypes = columnTypes.stream() + .map(converter::toStorageType) + .collect(toImmutableList()); + List columnNames = columnIds.stream().map(Object::toString).collect(toImmutableList()); + + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { + Map userMetadata = ImmutableMap.of(); + if (writeMetadata) { + ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); + for (int i = 0; i < columnIds.size(); i++) { + columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); + } + userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(columnTypesMap.build()))); + } + + orcWriter = new OrcWriter( + new OutputStreamOrcDataSink(new FileOutputStream(target)), + columnNames, + storageTypes, + ORC, + SNAPPY, + DEFAULT_OPTION, + userMetadata, + DEFAULT_STORAGE_TIMEZONE, + validate, + HASHED, + stats); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); + } + } + + @Override + public void appendPages(List pages) + { + for (Page page : pages) { + try { + orcWriter.write(page); + } + catch (IOException | UncheckedIOException e) { + throw new PrestoException(RAPTOR_WRITER_DATA_ERROR, e); + } + uncompressedSize += page.getLogicalSizeInBytes(); + rowCount += page.getPositionCount(); + } + } + + @Override + public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) + { + checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); + for (int i = 0; i < pageIndexes.length; i++) { + Page page = inputPages.get(pageIndexes[i]); + // This will do data copy; be aware + Page singleValuePage = page.getSingleValuePage(positionIndexes[i]); + try { + orcWriter.write(singleValuePage); + uncompressedSize += singleValuePage.getLogicalSizeInBytes(); + rowCount++; + } + catch (IOException | UncheckedIOException e) { + throw new PrestoException(RAPTOR_WRITER_DATA_ERROR, e); + } + } + } + + @Override + public void close() + throws IOException + { + if (closed) { + return; + } + closed = true; + + orcWriter.close(); + } + + @Override + public long getRowCount() + { + return rowCount; + } + + @Override + public long getUncompressedSize() + { + return uncompressedSize; + } + + private static boolean isUnique(Collection items) + { + return new HashSet<>(items).size() == items.size(); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java index d690c8e596521..34e8930987967 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.TimestampWithTimeZoneType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.spi.type.VarbinaryType; @@ -296,6 +297,9 @@ private static StorageType toStorageType(Type type) DecimalType decimalType = (DecimalType) type; return StorageType.decimal(decimalType.getPrecision(), decimalType.getScale()); } + if (type instanceof TimestampWithTimeZoneType) { + throw new PrestoException(NOT_SUPPORTED, "The current ORC writer does not support type: " + type); + } Class javaType = type.getJavaType(); if (javaType == boolean.class) { return StorageType.BOOLEAN; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index b7a6dd1803fb0..8f17cb7743941 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -19,6 +19,7 @@ import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.orc.TupleDomainOrcPredicate; import com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; import com.facebook.presto.orc.metadata.OrcType; @@ -32,6 +33,7 @@ import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.metadata.ShardRecorder; import com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo; +import com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.Page; @@ -55,6 +57,7 @@ import io.airlift.slice.XxHash64; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import org.joda.time.DateTimeZone; import javax.annotation.PreDestroy; import javax.inject.Inject; @@ -99,6 +102,8 @@ import static com.facebook.presto.raptor.storage.OrcPageSource.ROWID_COLUMN; import static com.facebook.presto.raptor.storage.OrcPageSource.SHARD_UUID_COLUMN; import static com.facebook.presto.raptor.storage.ShardStats.computeColumnStats; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.CharType.createCharType; @@ -151,9 +156,11 @@ public class OrcStorageManager private final long maxShardRows; private final DataSize maxShardSize; private final DataSize minAvailableSpace; + private final OrcOptimizedWriterStage orcOptimizedWriterStage; private final TypeManager typeManager; private final ExecutorService deletionExecutor; private final ExecutorService commitExecutor; + private final OrcWriterStats stats = new OrcWriterStats(); @Inject public OrcStorageManager( @@ -181,7 +188,8 @@ public OrcStorageManager( config.getShardRecoveryTimeout(), config.getMaxShardRows(), config.getMaxShardSize(), - config.getMinAvailableSpace()); + config.getMinAvailableSpace(), + config.getOrcOptimizedWriterStage()); } public OrcStorageManager( @@ -198,7 +206,8 @@ public OrcStorageManager( Duration shardRecoveryTimeout, long maxShardRows, DataSize maxShardSize, - DataSize minAvailableSpace) + DataSize minAvailableSpace, + OrcOptimizedWriterStage orcOptimizedWriterStage) { this.nodeId = requireNonNull(nodeId, "nodeId is null"); this.storageService = requireNonNull(storageService, "storageService is null"); @@ -217,6 +226,7 @@ public OrcStorageManager( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.deletionExecutor = newFixedThreadPool(deletionThreads, daemonThreadsNamed("raptor-delete-" + connectorId + "-%s")); this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); + this.orcOptimizedWriterStage = requireNonNull(orcOptimizedWriterStage, "orcOptimizedWriterStage is null"); } @PreDestroy @@ -573,7 +583,11 @@ private class OrcStoragePageSink private FileWriter writer; private UUID shardUuid; - public OrcStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) + public OrcStoragePageSink( + long transactionId, + List columnIds, + List columnTypes, + OptionalInt bucketNumber) { this.transactionId = transactionId; this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); @@ -688,7 +702,12 @@ private void createWriterIfNecessary() File stagingFile = storageService.getStagingFile(shardUuid); storageService.createParents(stagingFile); stagingFiles.add(stagingFile); - writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile); + if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { + writer = new OrcFileWriter(columnIds, columnTypes, stagingFile, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager); + } + else { + writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile); + } } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java index aeaee6ea3e800..0de7002784204 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.DateType; import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.TimeType; import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; @@ -74,6 +75,7 @@ private static ColumnStats doComputeColumnStats(OrcReader orcReader, long column } if (type.equals(BigintType.BIGINT) || type.equals(DateType.DATE) || + type.equals(TimeType.TIME) || type.equals(TimestampType.TIMESTAMP)) { return indexLong(type, reader, columnIndex, columnId); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java index 70d930c811496..ead02214313c9 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java @@ -33,6 +33,7 @@ import java.util.TimeZone; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.max; @@ -53,6 +54,7 @@ public class StorageManagerConfig private DataSize orcStreamBufferSize = new DataSize(8, MEGABYTE); private DataSize orcTinyStripeThreshold = new DataSize(8, MEGABYTE); private boolean orcLazyReadSmallRanges = true; + private OrcOptimizedWriterStage orcOptimizedWriterStage = DISABLED; private int deletionThreads = max(1, getRuntime().availableProcessors() / 2); private int recoveryThreads = 10; private int organizationThreads = 5; @@ -161,6 +163,23 @@ public StorageManagerConfig setOrcLazyReadSmallRanges(boolean orcLazyReadSmallRa return this; } + public enum OrcOptimizedWriterStage + { + DISABLED, ENABLED, ENABLED_AND_VALIDATED + } + + public OrcOptimizedWriterStage getOrcOptimizedWriterStage() + { + return orcOptimizedWriterStage; + } + + @Config("storage.orc.optimized-writer-stage") + public StorageManagerConfig setOrcOptimizedWriterStage(OrcOptimizedWriterStage orcOptimizedWriterStage) + { + this.orcOptimizedWriterStage = orcOptimizedWriterStage; + return this; + } + @Min(1) public int getDeletionThreads() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java new file mode 100644 index 0000000000000..71ff53c91088f --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignatureParameter; +import com.google.common.collect.ImmutableList; + +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.Chars.isCharType; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.Decimals.isLongDecimal; +import static com.facebook.presto.spi.type.Decimals.isShortDecimal; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimeType.TIME; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class StorageTypeConverter +{ + private final TypeManager typeManager; + + public StorageTypeConverter(TypeManager typeManager) + { + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + public Type toStorageType(Type type) + { + Type storageType; + if (type.equals(BOOLEAN) || + type.equals(BIGINT) || + type.equals(INTEGER) || + type.equals(SMALLINT) || + type.equals(TINYINT) || + type.equals(DOUBLE) || + type.equals(REAL) || + type.equals(DATE) || + isCharType(type) || + isVarcharType(type) || + isLongDecimal(type) || + isShortDecimal(type) || + type.equals(VARBINARY)) { + storageType = type; + } + // Raptor does not handle TIME or TIMESTAMP timezone conversion. + // The timezone used by ORC reader/writer is a placeholder that is not used. + // Check OrcType::toOrcType() for reference. + else if (type.equals(TIME) || type.equals(TIMESTAMP)) { + storageType = BIGINT; + } + else if (type instanceof ArrayType) { + storageType = new ArrayType(toStorageType(((ArrayType) type).getElementType())); + } + else if (type instanceof MapType) { + storageType = mapType(toStorageType(((MapType) type).getKeyType()), toStorageType(((MapType) type).getValueType())); + } + else { + throw new PrestoException(NOT_SUPPORTED, "Type not supported: " + type); + } + + // We cannot write different java types because when Raptor calculates stats, it uses the column types rather than storage types. + // Need to make sure min/max are compliant with both storage and column types. + checkState(storageType.getJavaType().equals(type.getJavaType())); + return storageType; + } + + private Type mapType(Type keyType, Type valueType) + { + return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( + TypeSignatureParameter.of(keyType.getTypeSignature()), + TypeSignatureParameter.of(valueType.getTypeSignature()))); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java index bd8426fffdc55..f7c453922d9d6 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java @@ -100,7 +100,7 @@ public void setup() NodeManager nodeManager = new TestingNodeManager(); NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; ShardManager shardManager = createShardManager(dbi); - StorageManager storageManager = createOrcStorageManager(dbi, dataDir); + StorageManager storageManager = createOrcStorageManager(dbi, dataDir, true); StorageManagerConfig config = new StorageManagerConfig(); connector = new RaptorConnector( new LifeCycleManager(ImmutableList.of(), null), diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java index 7b3da44b5f38f..50f7d2c4c5bab 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java @@ -18,7 +18,9 @@ import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import org.joda.time.DateTimeZone; @@ -96,4 +98,12 @@ public static byte octet(int b) checkArgument((b >= 0) && (b <= 0xFF), "octet not in range: %s", b); return (byte) b; } + + public static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean useOptimizedOrcWriter) + { + if (useOptimizedOrcWriter) { + return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), new TypeRegistry()); + } + return new OrcRecordWriter(columnIds, columnTypes, file, true); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java index 4295588cce08e..355dc056eea7f 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java @@ -17,6 +17,7 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; @@ -34,6 +35,7 @@ import io.airlift.json.JsonCodec; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; @@ -71,6 +73,12 @@ public class TestOrcFileRewriter private File temporary; + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeClass public void setup() { @@ -84,8 +92,8 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test - public void testRewrite() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewrite(boolean useOptimizedOrcWriter) throws Exception { TypeManager typeManager = new TypeRegistry(); @@ -103,7 +111,7 @@ public void testRewrite() List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")) .row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")) @@ -264,15 +272,15 @@ public void testRewrite() } } - @Test - public void testRewriteWithoutMetadata() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file, false)) { + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello") .row(777L, "sky") @@ -339,15 +347,15 @@ public void testRewriteWithoutMetadata() } } - @Test - public void testRewriteAllRowsDeleted() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriteAllRowsDeleted(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(3L); List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } @@ -363,15 +371,15 @@ public void testRewriteAllRowsDeleted() assertFalse(newFile.exists()); } - @Test - public void testRewriteNoRowsDeleted() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriteNoRowsDeleted(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(3L); List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } @@ -382,18 +390,21 @@ public void testRewriteNoRowsDeleted() assertEquals(info.getRowCount(), 2); assertEquals(info.getUncompressedSize(), 16); - assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath())); + // TODO: enable check for optimized ORC writer once the rewriter uses the optimized ORC writer as well + if (!useOptimizedOrcWriter) { + assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath())); + } } - @Test - public void testUncompressedSize() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testUncompressedSize(boolean useOptimizedOrcWriter) throws Exception { List columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L); List columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { List pages = rowPagesBuilder(columnTypes) .row(true, 123L, 98.7, "hello", utf8Slice("abc")) .row(false, 456L, 65.4, "world", utf8Slice("xyz")) @@ -407,4 +418,12 @@ public void testUncompressedSize() assertEquals(info.getRowCount(), 3); assertEquals(info.getUncompressedSize(), 55); } + + private static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean writeMetadata, boolean useOptimizedOrcWriter) + { + if (useOptimizedOrcWriter) { + return new OrcFileWriter(columnIds, columnTypes, file, writeMetadata, true, new OrcWriterStats(), new TypeRegistry()); + } + return new OrcRecordWriter(columnIds, columnTypes, file, writeMetadata); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java index a64f9c52027ca..6f96e0a04bcc4 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java @@ -32,6 +32,7 @@ import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.SqlDate; +import com.facebook.presto.spi.type.SqlTime; import com.facebook.presto.spi.type.SqlTimestamp; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.Type; @@ -52,6 +53,7 @@ import org.skife.jdbi.v2.IDBI; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; @@ -72,10 +74,13 @@ import static com.facebook.presto.raptor.storage.OrcStorageManager.xxhash64; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.octets; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.TimeType.TIME; import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; @@ -97,6 +102,7 @@ import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.joda.time.DateTimeZone.UTC; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; @@ -130,6 +136,12 @@ public class TestOrcStorageManager private Optional backupStore; private InMemoryShardRecorder shardRecorder; + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeMethod public void setup() { @@ -164,11 +176,11 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test - public void testWriter() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testWriter(boolean useOptimizedOrcWriter) throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -237,11 +249,11 @@ public void testWriter() } } - @Test - public void testReader() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testReader(boolean useOptimizedOrcWriter) throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); List columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE); @@ -311,11 +323,11 @@ public void testReader() } } - @Test - public void testRewriter() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriter(boolean useOptimizedOrcWriter) throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); long transactionId = TRANSACTION_ID; List columnIds = ImmutableList.of(3L, 7L); @@ -356,8 +368,8 @@ public void testRewriter() assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid()); } - @Test - public void testWriterRollback() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testWriterRollback(boolean useOptimizedOrcWriter) { // verify staging directory is empty File staging = new File(new File(temporary, "data"), "staging"); @@ -365,7 +377,7 @@ public void testWriterRollback() assertEquals(staging.list(), new String[] {}); // create a shard in staging - OrcStorageManager manager = createOrcStorageManager(); + OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -500,9 +512,20 @@ public void testShardStatsDateTimestamp() } @Test - public void testMaxShardRows() + public void testShardStatsTime() { - OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE)); + long minTime = sqlTime(2004, 8, 22).getMillis(); + long maxTime = sqlTime(2006, 4, 22).getMillis(); + + // Apache ORC writer does not support TIME + List columnStats = columnStats(types(TIME), row(minTime), row(maxTime)); + assertColumnStats(columnStats, 1, minTime, maxTime); + } + + @Test(dataProvider = "useOptimizedOrcWriter") + public void testMaxShardRows(boolean useOptimizedOrcWriter) + { + OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE), useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -516,8 +539,8 @@ public void testMaxShardRows() assertTrue(sink.isFull()); } - @Test - public void testMaxFileSize() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testMaxFileSize(boolean useOptimizedOrcWriter) { List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(5)); @@ -528,7 +551,7 @@ public void testMaxFileSize() .build(); // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full - OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE)); + OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE), useOptimizedOrcWriter); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(pages); assertTrue(sink.isFull()); @@ -550,22 +573,22 @@ private static StoragePageSink createStoragePageSink(StorageManager manager, Lis return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); } - private OrcStorageManager createOrcStorageManager() + private OrcStorageManager createOrcStorageManager(boolean useOptimizedWriter) { - return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); + return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE, useOptimizedWriter); } - private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize) + private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize, boolean useOptimizedWriter) { - return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); + return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize, useOptimizedWriter); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary) + public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, boolean useOptimizedWriter) { - return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); + return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedWriter); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows) + public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows, boolean useOptimizedWriter) { File directory = new File(temporary, "data"); StorageService storageService = new FileStorageService(directory); @@ -590,7 +613,8 @@ public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary recoveryManager, new InMemoryShardRecorder(), maxShardRows, - MAX_FILE_SIZE); + MAX_FILE_SIZE, + useOptimizedWriter); } public static OrcStorageManager createOrcStorageManager( @@ -599,7 +623,8 @@ public static OrcStorageManager createOrcStorageManager( ShardRecoveryManager recoveryManager, ShardRecorder shardRecorder, int maxShardRows, - DataSize maxFileSize) + DataSize maxFileSize, + boolean useOptimizedWriter) { return new OrcStorageManager( CURRENT_NODE, @@ -615,7 +640,8 @@ public static OrcStorageManager createOrcStorageManager( SHARD_RECOVERY_TIMEOUT, maxShardRows, maxFileSize, - new DataSize(0, BYTE)); + new DataSize(0, BYTE), + useOptimizedWriter ? ENABLED_AND_VALIDATED : DISABLED); } private static void assertFileEquals(File actual, File expected) @@ -661,7 +687,15 @@ private List columnStats(List columnTypes, Object[]... rows) } List columnIds = list.build(); - OrcStorageManager manager = createOrcStorageManager(); + List apacheOrcWriterStats = columnStats(false, columnIds, columnTypes, rows); + List optimizedOrcWriterStats = columnStats(true, columnIds, columnTypes, rows); + assertEquals(apacheOrcWriterStats, optimizedOrcWriterStats); + return optimizedOrcWriterStats; + } + + private List columnStats(boolean useOptimizedWriter, List columnIds, List columnTypes, Object[]... rows) + { + OrcStorageManager manager = createOrcStorageManager(useOptimizedWriter); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build()); List shards = getFutureValue(sink.commit()); @@ -681,6 +715,12 @@ private static SqlDate sqlDate(int year, int month, int day) return new SqlDate(Days.daysBetween(EPOCH, date).getDays()); } + private static SqlTime sqlTime(int year, int month, int day) + { + DateTime date = new DateTime(year, month, day, 0, 0, 0, 0, UTC); + return new SqlTime(NANOSECONDS.toMillis(date.toLocalTime().getMillisOfDay())); + } + private static SqlTimestamp sqlTimestamp(int year, int month, int day, int hour, int minute, int second) { return sqlTimestampOf(year, month, day, hour, minute, second, 0, UTC, UTC_KEY, SESSION); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java index 88bf81bedc6dd..178aee64d00d8 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java @@ -18,9 +18,13 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.SqlDate; +import com.facebook.presto.spi.type.SqlTime; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -31,22 +35,34 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.json.JsonCodec; +import org.joda.time.DateTime; +import org.joda.time.Days; +import org.joda.time.chrono.ISOChronology; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; import java.util.List; +import static com.facebook.presto.raptor.storage.OrcTestingUtil.createFileWriter; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReaderNoRows; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; import static com.facebook.presto.raptor.storage.OrcTestingUtil.octets; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.TimeType.TIME; +import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; +import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.tests.StructuralTestUtil.arrayBlockOf; import static com.facebook.presto.tests.StructuralTestUtil.arrayBlocksEqual; import static com.facebook.presto.tests.StructuralTestUtil.mapBlockOf; @@ -57,9 +73,13 @@ import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; +import static java.util.Locale.ENGLISH; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.joda.time.DateTimeZone.UTC; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class TestShardWriter { @@ -67,6 +87,12 @@ public class TestShardWriter private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeClass public void setup() { @@ -80,33 +106,37 @@ public void tearDown() deleteRecursively(directory.toPath(), ALLOW_INSECURE); } - @Test - public void testWriter() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testWriter(boolean useOptimizedOrcWriter) throws Exception { TypeManager typeManager = new TypeRegistry(); // associate typeManager with a function manager new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - List columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L); + List columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L); ArrayType arrayType = new ArrayType(BIGINT); ArrayType arrayOfArrayType = new ArrayType(arrayType); Type mapType = typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( TypeSignatureParameter.of(createVarcharType(10).getTypeSignature()), TypeSignatureParameter.of(BOOLEAN.getTypeSignature()))); - List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType); + List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType, TIMESTAMP, TIME, DATE); File file = new File(directory, System.nanoTime() + ".orc"); byte[] bytes1 = octets(0x00, 0xFE, 0xFF); byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); + long timestampValue = sqlTimestampOf(2002, 4, 6, 7, 8, 9, 0, UTC, UTC_KEY, SESSION).getMillisUtc(); + long timeValue = new SqlTime(NANOSECONDS.toMillis(new DateTime(2004, 11, 29, 0, 0, 0, 0, UTC).toLocalTime().getMillisOfDay())).getMillis(); + DateTime date = new DateTime(2001, 11, 22, 0, 0, 0, 0, UTC); + int dateValue = new SqlDate(Days.daysBetween(new DateTime(0, ISOChronology.getInstanceUTC()), date).getDays()).getDays(); RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes) - .row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))) - .row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))) - .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))); + .row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), timestampValue, timeValue, dateValue) + .row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)), timestampValue, timeValue, dateValue) + .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)), timestampValue, timeValue, dateValue); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader()); - FileWriter writer = new OrcRecordWriter(columnIds, columnTypes, file)) { + FileWriter writer = createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { writer.appendPages(rowPagesBuilder.build()); } @@ -176,6 +206,21 @@ public void testWriter() assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))); assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)))); + Block column8 = reader.readBlock(TIMESTAMP, 8); + assertEquals(TIMESTAMP.getLong(column8, 0), timestampValue); + assertEquals(TIMESTAMP.getLong(column8, 1), timestampValue); + assertEquals(TIMESTAMP.getLong(column8, 2), timestampValue); + + Block column9 = reader.readBlock(TIME, 9); + assertEquals(TIME.getLong(column9, 0), timeValue); + assertEquals(TIME.getLong(column9, 1), timeValue); + assertEquals(TIME.getLong(column9, 2), timeValue); + + Block column10 = reader.readBlock(DATE, 10); + assertEquals(DATE.getLong(column10, 0), dateValue); + assertEquals(DATE.getLong(column10, 1), dateValue); + assertEquals(DATE.getLong(column10, 2), dateValue); + assertEquals(reader.nextBatch(), -1); assertEquals(reader.getReaderPosition(), 3); assertEquals(reader.getFilePosition(), reader.getFilePosition()); @@ -190,11 +235,25 @@ public void testWriter() .put(8L, arrayType.getTypeSignature()) .put(9L, mapType.getTypeSignature()) .put(10L, arrayOfArrayType.getTypeSignature()) + .put(11L, TIMESTAMP.getTypeSignature()) + .put(12L, TIME.getTypeSignature()) + .put(13L, DATE.getTypeSignature()) .build())); } File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc"); assertFalse(crcFile.exists()); + + // Test unsupported types + for (Type type : ImmutableList.of(TIMESTAMP_WITH_TIME_ZONE, RowType.anonymous(ImmutableList.of(BIGINT, DOUBLE)))) { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader())) { + createFileWriter(ImmutableList.of(1L), ImmutableList.of(type), file, useOptimizedOrcWriter); + fail(); + } + catch (PrestoException e) { + assertTrue(e.getMessage().toLowerCase(ENGLISH).contains("type")); + } + } } @SuppressWarnings("EmptyTryBlock") @@ -207,7 +266,7 @@ public void testWriterZeroRows() File file = new File(directory, System.nanoTime() + ".orc"); - try (OrcRecordWriter ignored = new OrcRecordWriter(columnIds, columnTypes, file)) { + try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file, false)) { // no rows } @@ -218,6 +277,19 @@ public void testWriterZeroRows() assertEquals(reader.nextBatch(), -1); } + + // optimized ORC writer will flush metadata on close + try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file, true)) { + // no rows + } + + try (OrcDataSource dataSource = fileOrcDataSource(file)) { + OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); + assertEquals(reader.getReaderRowCount(), 0); + assertEquals(reader.getReaderPosition(), 0); + + assertEquals(reader.nextBatch(), -1); + } } @SuppressWarnings("EmptyClass") diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java index 6ee890e6fade6..52c526a5cbf82 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java @@ -24,6 +24,8 @@ import java.io.File; import java.util.Map; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; +import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; @@ -52,6 +54,7 @@ public void testDefaults() .setOrcStreamBufferSize(new DataSize(8, MEGABYTE)) .setOrcTinyStripeThreshold(new DataSize(8, MEGABYTE)) .setOrcLazyReadSmallRanges(true) + .setOrcOptimizedWriterStage(DISABLED) .setDeletionThreads(max(1, getRuntime().availableProcessors() / 2)) .setShardRecoveryTimeout(new Duration(30, SECONDS)) .setMissingShardDiscoveryInterval(new Duration(5, MINUTES)) @@ -81,6 +84,7 @@ public void testExplicitPropertyMappings() .put("storage.orc.stream-buffer-size", "16kB") .put("storage.orc.tiny-stripe-threshold", "15kB") .put("storage.orc.lazy-read-small-ranges", "false") + .put("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED") .put("storage.max-deletion-threads", "999") .put("storage.shard-recovery-timeout", "1m") .put("storage.missing-shard-discovery-interval", "4m") @@ -107,6 +111,7 @@ public void testExplicitPropertyMappings() .setOrcStreamBufferSize(new DataSize(16, KILOBYTE)) .setOrcTinyStripeThreshold(new DataSize(15, KILOBYTE)) .setOrcLazyReadSmallRanges(false) + .setOrcOptimizedWriterStage(ENABLED_AND_VALIDATED) .setDeletionThreads(999) .setShardRecoveryTimeout(new Duration(1, MINUTES)) .setMissingShardDiscoveryInterval(new Duration(4, MINUTES)) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java index 3a54275729037..9cc835ba8bca6 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java @@ -18,7 +18,6 @@ import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ShardInfo; -import com.facebook.presto.raptor.storage.OrcStorageManager; import com.facebook.presto.raptor.storage.ReaderAttributes; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.storage.StoragePageSink; @@ -38,6 +37,7 @@ import org.skife.jdbi.v2.IDBI; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; @@ -74,19 +74,22 @@ public class TestShardCompactor private static final PagesIndexPageSorter PAGE_SORTER = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - private OrcStorageManager storageManager; - private ShardCompactor compactor; private File temporary; + private IDBI dbi; private Handle dummyHandle; + @DataProvider(name = "useOptimizedOrcWriter") + public static Object[][] useOptimizedOrcWriter() + { + return new Object[][] {{true}, {false}}; + } + @BeforeMethod public void setup() { temporary = createTempDir(); - IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); dummyHandle = dbi.open(); - storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); - compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES); } @AfterMethod(alwaysRun = true) @@ -99,10 +102,11 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test - public void testShardCompactor() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testShardCompactor(boolean useOptimizedOrcWriter) throws Exception { + StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedOrcWriter); List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP); @@ -117,17 +121,19 @@ public void testShardCompactor() Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); long transactionId = 1; + ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES); List outputShards = compactor.compact(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes)); assertEquals(outputShards.size(), expectedOutputShards); Set outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); - assertShardEqualsIgnoreOrder(inputUuids, outputUuids, columnIds, columnTypes); + assertShardEqualsIgnoreOrder(storageManager, inputUuids, outputUuids, columnIds, columnTypes); } - @Test - public void testShardCompactorSorted() + @Test(dataProvider = "useOptimizedOrcWriter") + public void testShardCompactorSorted(boolean useOptimizedOrcWriter) throws Exception { + StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedOrcWriter); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP, DOUBLE); List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); List sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L); @@ -145,13 +151,14 @@ public void testShardCompactorSorted() Set inputUuids = inputShards.stream().map(ShardInfo::getShardUuid).collect(toSet()); long transactionId = 1; + ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES); List outputShards = compactor.compactSorted(transactionId, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes), sortColumnIds, sortOrders); List outputUuids = outputShards.stream() .map(ShardInfo::getShardUuid) .collect(toList()); assertEquals(outputShards.size(), expectedOutputShards); - assertShardEqualsSorted(inputUuids, outputUuids, columnIds, columnTypes, sortIndexes, sortOrders); + assertShardEqualsSorted(storageManager, inputUuids, outputUuids, columnIds, columnTypes, sortIndexes, sortOrders); } private static long computeExpectedOutputShards(long totalRows) @@ -159,23 +166,23 @@ private static long computeExpectedOutputShards(long totalRows) return ((totalRows % MAX_SHARD_ROWS) != 0) ? ((totalRows / MAX_SHARD_ROWS) + 1) : (totalRows / MAX_SHARD_ROWS); } - private void assertShardEqualsIgnoreOrder(Set inputUuids, Set outputUuids, List columnIds, List columnTypes) + private void assertShardEqualsIgnoreOrder(StorageManager storageManager, Set inputUuids, Set outputUuids, List columnIds, List columnTypes) throws IOException { - MaterializedResult inputRows = getMaterializedRows(ImmutableList.copyOf(inputUuids), columnIds, columnTypes); - MaterializedResult outputRows = getMaterializedRows(ImmutableList.copyOf(outputUuids), columnIds, columnTypes); + MaterializedResult inputRows = getMaterializedRows(storageManager, ImmutableList.copyOf(inputUuids), columnIds, columnTypes); + MaterializedResult outputRows = getMaterializedRows(storageManager, ImmutableList.copyOf(outputUuids), columnIds, columnTypes); assertEqualsIgnoreOrder(outputRows, inputRows); } - private void assertShardEqualsSorted(Set inputUuids, List outputUuids, List columnIds, List columnTypes, List sortIndexes, List sortOrders) + private void assertShardEqualsSorted(StorageManager storageManager, Set inputUuids, List outputUuids, List columnIds, List columnTypes, List sortIndexes, List sortOrders) throws IOException { - List inputPages = getPages(inputUuids, columnIds, columnTypes); + List inputPages = getPages(storageManager, inputUuids, columnIds, columnTypes); List sortTypes = sortIndexes.stream().map(columnTypes::get).collect(toList()); MaterializedResult inputRowsSorted = sortAndMaterialize(inputPages, columnTypes, sortIndexes, sortOrders, sortTypes); - MaterializedResult outputRows = extractColumns(getMaterializedRows(outputUuids, columnIds, columnTypes), sortIndexes, sortTypes); + MaterializedResult outputRows = extractColumns(getMaterializedRows(storageManager, outputUuids, columnIds, columnTypes), sortIndexes, sortTypes); assertEquals(outputRows, inputRowsSorted); } @@ -223,12 +230,12 @@ private static MaterializedResult sortAndMaterialize(List pages, List getPages(Set uuids, List columnIds, List columnTypes) + private List getPages(StorageManager storageManager, Set uuids, List columnIds, List columnTypes) throws IOException { ImmutableList.Builder pages = ImmutableList.builder(); for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { + try (ConnectorPageSource pageSource = getPageSource(storageManager, columnIds, columnTypes, uuid)) { while (!pageSource.isFinished()) { Page outputPage = pageSource.getNextPage(); if (outputPage == null) { @@ -241,12 +248,12 @@ private List getPages(Set uuids, List columnIds, List co return pages.build(); } - private MaterializedResult getMaterializedRows(List uuids, List columnIds, List columnTypes) + private MaterializedResult getMaterializedRows(StorageManager storageManager, List uuids, List columnIds, List columnTypes) throws IOException { MaterializedResult.Builder rows = MaterializedResult.resultBuilder(SESSION, columnTypes); for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = getPageSource(columnIds, columnTypes, uuid)) { + try (ConnectorPageSource pageSource = getPageSource(storageManager, columnIds, columnTypes, uuid)) { MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes); rows.rows(result.getMaterializedRows()); } @@ -254,7 +261,7 @@ private MaterializedResult getMaterializedRows(List uuids, List colu return rows.build(); } - private ConnectorPageSource getPageSource(List columnIds, List columnTypes, UUID uuid) + private ConnectorPageSource getPageSource(StorageManager storageManager, List columnIds, List columnTypes, UUID uuid) { return storageManager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, TupleDomain.all(), READER_ATTRIBUTES); } From c070f5bc6d6951505d9b3c4b9115cb23e3667003 Mon Sep 17 00:00:00 2001 From: James Sun Date: Sat, 6 Apr 2019 23:37:32 -0700 Subject: [PATCH 23/66] Abstract out Raptor FileRewriter --- .../presto/raptor/storage/FileRewriter.java | 23 +++++++++++ .../presto/raptor/storage/OrcFileInfo.java | 39 +++++++++++++++++++ ...writer.java => OrcRecordFileRewriter.java} | 32 +++------------ .../raptor/storage/OrcStorageManager.java | 13 +++++-- .../raptor/storage/TestOrcFileRewriter.java | 11 +++--- 5 files changed, 82 insertions(+), 36 deletions(-) create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java rename presto-raptor/src/main/java/com/facebook/presto/raptor/storage/{OrcFileRewriter.java => OrcRecordFileRewriter.java} (91%) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java new file mode 100644 index 0000000000000..291ce41cef2b1 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import java.io.File; +import java.io.IOException; +import java.util.BitSet; + +public interface FileRewriter +{ + OrcFileInfo rewrite(File input, File output, BitSet rowsToDelete) throws IOException; +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java new file mode 100644 index 0000000000000..2a96cec4023da --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileInfo.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import javax.annotation.concurrent.Immutable; + +@Immutable +public class OrcFileInfo +{ + private final long rowCount; + private final long uncompressedSize; + + public OrcFileInfo(long rowCount, long uncompressedSize) + { + this.rowCount = rowCount; + this.uncompressedSize = uncompressedSize; + } + + public long getRowCount() + { + return rowCount; + } + + public long getUncompressedSize() + { + return uncompressedSize; + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java similarity index 91% rename from presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java rename to presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java index 9b7ed952b0f1d..5c70eaf820c40 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java @@ -61,14 +61,14 @@ import static org.apache.hadoop.hive.ql.io.orc.OrcFile.createWriter; import static org.apache.hadoop.hive.ql.io.orc.OrcUtil.getFieldValue; -public final class OrcFileRewriter +public final class OrcRecordFileRewriter + implements FileRewriter { - private static final Logger log = Logger.get(OrcFileRewriter.class); + private static final Logger log = Logger.get(OrcRecordFileRewriter.class); private static final Configuration CONFIGURATION = new Configuration(); - private OrcFileRewriter() {} - - public static OrcFileInfo rewrite(File input, File output, BitSet rowsToDelete) + @Override + public OrcFileInfo rewrite(File input, File output, BitSet rowsToDelete) throws IOException { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); @@ -210,26 +210,4 @@ private static int uncompressedSize(Object object) } throw new IOException("Unhandled ORC object: " + object.getClass().getName()); } - - public static class OrcFileInfo - { - private final long rowCount; - private final long uncompressedSize; - - public OrcFileInfo(long rowCount, long uncompressedSize) - { - this.rowCount = rowCount; - this.uncompressedSize = uncompressedSize; - } - - public long getRowCount() - { - return rowCount; - } - - public long getUncompressedSize() - { - return uncompressedSize; - } - } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index 8f17cb7743941..e07cb749a71b7 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -32,7 +32,6 @@ import com.facebook.presto.raptor.metadata.ShardDelta; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.metadata.ShardRecorder; -import com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo; import com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.NodeManager; @@ -160,6 +159,7 @@ public class OrcStorageManager private final TypeManager typeManager; private final ExecutorService deletionExecutor; private final ExecutorService commitExecutor; + private final FileRewriter fileRewriter; private final OrcWriterStats stats = new OrcWriterStats(); @Inject @@ -227,6 +227,13 @@ public OrcStorageManager( this.deletionExecutor = newFixedThreadPool(deletionThreads, daemonThreadsNamed("raptor-delete-" + connectorId + "-%s")); this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); this.orcOptimizedWriterStage = requireNonNull(orcOptimizedWriterStage, "orcOptimizedWriterStage is null"); + if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { + // TODO: add new rewriter. + this.fileRewriter = null; + } + else { + this.fileRewriter = new OrcRecordFileRewriter(); + } } @PreDestroy @@ -446,10 +453,10 @@ private static Collection shardDelta(UUID oldShardUuid, Optional Date: Sat, 6 Apr 2019 23:37:34 -0700 Subject: [PATCH 24/66] Add column types to Raptor split The optimized ORC writer will serve as rewriter for Raptor. However it needs the schema of the table in order to read/write data. Encode the schema info in splits. --- .../presto/raptor/RaptorMetadata.java | 5 ++++ .../facebook/presto/raptor/RaptorSplit.java | 29 ++++++++++++++----- .../presto/raptor/RaptorSplitManager.java | 10 +++++-- .../presto/raptor/RaptorTableHandle.java | 11 +++++++ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java index c89ceee8c5531..14772f806e9d1 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java @@ -75,6 +75,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongConsumer; +import java.util.stream.Collectors; import static com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType; import static com.facebook.presto.raptor.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME; @@ -188,6 +189,7 @@ private RaptorTableHandle getTableHandle(SchemaTableName tableName) table.getBucketCount(), table.isOrganized(), OptionalLong.empty(), + Optional.empty(), false); } @@ -775,6 +777,8 @@ public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTable setTransactionId(transactionId); + Map columnTypes = dao.listTableColumns(handle.getTableId()).stream().collect(Collectors.toMap(k -> String.valueOf(k.getColumnId()), TableColumn::getDataType)); + return new RaptorTableHandle( connectorId, handle.getSchemaName(), @@ -785,6 +789,7 @@ public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTable handle.getBucketCount(), handle.isOrganized(), OptionalLong.of(transactionId), + Optional.of(columnTypes), true); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java index c0ad106e324b3..1a9433539f300 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplit.java @@ -16,12 +16,15 @@ import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; @@ -39,6 +42,7 @@ public class RaptorSplit private final List addresses; private final TupleDomain effectivePredicate; private final OptionalLong transactionId; + private final Optional> columnTypes; @JsonCreator public RaptorSplit( @@ -46,9 +50,10 @@ public RaptorSplit( @JsonProperty("shardUuids") Set shardUuids, @JsonProperty("bucketNumber") OptionalInt bucketNumber, @JsonProperty("effectivePredicate") TupleDomain effectivePredicate, - @JsonProperty("transactionId") OptionalLong transactionId) + @JsonProperty("transactionId") OptionalLong transactionId, + @JsonProperty("columnTypes") Optional> columnTypes) { - this(connectorId, shardUuids, bucketNumber, ImmutableList.of(), effectivePredicate, transactionId); + this(connectorId, shardUuids, bucketNumber, ImmutableList.of(), effectivePredicate, transactionId, columnTypes); } public RaptorSplit( @@ -56,9 +61,10 @@ public RaptorSplit( UUID shardUuid, List addresses, TupleDomain effectivePredicate, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> columnTypes) { - this(connectorId, ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses, effectivePredicate, transactionId); + this(connectorId, ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses, effectivePredicate, transactionId, columnTypes); } public RaptorSplit( @@ -67,9 +73,10 @@ public RaptorSplit( int bucketNumber, HostAddress address, TupleDomain effectivePredicate, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> columnTypes) { - this(connectorId, shardUuids, OptionalInt.of(bucketNumber), ImmutableList.of(address), effectivePredicate, transactionId); + this(connectorId, shardUuids, OptionalInt.of(bucketNumber), ImmutableList.of(address), effectivePredicate, transactionId, columnTypes); } private RaptorSplit( @@ -78,7 +85,8 @@ private RaptorSplit( OptionalInt bucketNumber, List addresses, TupleDomain effectivePredicate, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> columnTypes) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.shardUuids = ImmutableSet.copyOf(requireNonNull(shardUuids, "shardUuid is null")); @@ -86,6 +94,7 @@ private RaptorSplit( this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null")); this.effectivePredicate = requireNonNull(effectivePredicate, "effectivePredicate is null"); this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); } @Override @@ -130,6 +139,12 @@ public OptionalLong getTransactionId() return transactionId; } + @JsonProperty + public Optional> getColumnTypes() + { + return columnTypes; + } + @Override public Object getInfo() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java index 0a3d0ad5b5884..720246302b50a 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; import org.skife.jdbi.v2.ResultIterator; @@ -106,7 +107,7 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, Co OptionalLong transactionId = table.getTransactionId(); Optional> bucketToNode = handle.getPartitioning().map(RaptorPartitioningHandle::getBucketToNode); verify(bucketed == bucketToNode.isPresent(), "mismatched bucketCount and bucketToNode presence"); - return new RaptorSplitSource(tableId, merged, effectivePredicate, transactionId, bucketToNode); + return new RaptorSplitSource(tableId, merged, effectivePredicate, transactionId, table.getColumnTypes(), bucketToNode); } private static List getAddressesForNodes(Map nodeMap, Iterable nodeIdentifiers) @@ -140,6 +141,7 @@ private class RaptorSplitSource private final long tableId; private final TupleDomain effectivePredicate; private final OptionalLong transactionId; + private final Optional> columnTypes; private final Optional> bucketToNode; private final ResultIterator iterator; @@ -151,11 +153,13 @@ public RaptorSplitSource( boolean merged, TupleDomain effectivePredicate, OptionalLong transactionId, + Optional> columnTypes, Optional> bucketToNode) { this.tableId = tableId; this.effectivePredicate = requireNonNull(effectivePredicate, "effectivePredicate is null"); this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.columnTypes = requireNonNull(columnTypes, "columnTypesis null"); this.bucketToNode = requireNonNull(bucketToNode, "bucketToNode is null"); ResultIterator iterator; @@ -238,7 +242,7 @@ private ConnectorSplit createSplit(BucketShards bucketShards) addresses = ImmutableList.of(node.getHostAndPort()); } - return new RaptorSplit(connectorId, shardId, addresses, effectivePredicate, transactionId); + return new RaptorSplit(connectorId, shardId, addresses, effectivePredicate, transactionId, columnTypes); } private ConnectorSplit createBucketSplit(int bucketNumber, Set shards) @@ -257,7 +261,7 @@ private ConnectorSplit createBucketSplit(int bucketNumber, Set shard .collect(toSet()); HostAddress address = node.getHostAndPort(); - return new RaptorSplit(connectorId, shardUuids, bucketNumber, address, effectivePredicate, transactionId); + return new RaptorSplit(connectorId, shardUuids, bucketNumber, address, effectivePredicate, transactionId, columnTypes); } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java index 736f820896b55..97a4bd49e6966 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java @@ -14,9 +14,11 @@ package com.facebook.presto.raptor; import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; @@ -39,6 +41,7 @@ public final class RaptorTableHandle private final OptionalInt bucketCount; private final boolean organized; private final OptionalLong transactionId; + private final Optional> columnTypes; private final boolean delete; @JsonCreator @@ -52,6 +55,7 @@ public RaptorTableHandle( @JsonProperty("bucketCount") OptionalInt bucketCount, @JsonProperty("organized") boolean organized, @JsonProperty("transactionId") OptionalLong transactionId, + @JsonProperty("columnTypes") Optional> columnTypes, @JsonProperty("delete") boolean delete) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); @@ -66,6 +70,7 @@ public RaptorTableHandle( this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); this.organized = organized; this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); this.delete = delete; } @@ -129,6 +134,12 @@ public OptionalLong getTransactionId() return transactionId; } + @JsonProperty + public Optional> getColumnTypes() + { + return columnTypes; + } + @JsonProperty public boolean isDelete() { From 8d643f4960f011c97345a3a884f1160405e71ce5 Mon Sep 17 00:00:00 2001 From: James Sun Date: Sat, 6 Apr 2019 23:37:35 -0700 Subject: [PATCH 25/66] Introduce OrcPageFileRewriter --- .../raptor/RaptorPageSourceProvider.java | 12 +- .../presto/raptor/storage/FileRewriter.java | 5 +- .../presto/raptor/storage/OrcFileWriter.java | 2 +- .../raptor/storage/OrcPageFileRewriter.java | 236 +++++++++++++ .../raptor/storage/OrcRecordFileRewriter.java | 3 +- .../raptor/storage/OrcStorageManager.java | 25 +- .../presto/raptor/storage/StorageManager.java | 7 +- .../presto/raptor/storage/OrcTestingUtil.java | 7 +- .../raptor/storage/TestOrcFileRewriter.java | 309 +++++++++++++++++- .../raptor/storage/TestOrcStorageManager.java | 9 +- 10 files changed, 578 insertions(+), 37 deletions(-) create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java index c6ab4f4492ad8..4438787ffa394 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java @@ -29,6 +29,8 @@ import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.UUID; @@ -56,14 +58,15 @@ public ConnectorPageSource createPageSource(ConnectorTransactionHandle transacti TupleDomain predicate = raptorSplit.getEffectivePredicate(); ReaderAttributes attributes = ReaderAttributes.from(session); OptionalLong transactionId = raptorSplit.getTransactionId(); + Optional> columnTypes = raptorSplit.getColumnTypes(); if (raptorSplit.getShardUuids().size() == 1) { UUID shardUuid = raptorSplit.getShardUuids().iterator().next(); - return createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId); + return createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes); } Iterator iterator = raptorSplit.getShardUuids().stream() - .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId)) + .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes)) .iterator(); return new ConcatPageSource(iterator); @@ -75,12 +78,13 @@ private ConnectorPageSource createPageSource( List columns, TupleDomain predicate, ReaderAttributes attributes, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> allColumnTypes) { List columnHandles = columns.stream().map(RaptorColumnHandle.class::cast).collect(toList()); List columnIds = columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); List columnTypes = columnHandles.stream().map(RaptorColumnHandle::getColumnType).collect(toList()); - return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, attributes, transactionId); + return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, attributes, transactionId, allColumnTypes); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java index 291ce41cef2b1..d99219339ba9f 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java @@ -13,11 +13,14 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.spi.type.Type; + import java.io.File; import java.io.IOException; import java.util.BitSet; +import java.util.Map; public interface FileRewriter { - OrcFileInfo rewrite(File input, File output, BitSet rowsToDelete) throws IOException; + OrcFileInfo rewrite(Map columns, File input, File output, BitSet rowsToDelete) throws IOException; } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java index e5dcae8cd7fa0..28332e976a819 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java @@ -51,8 +51,8 @@ public class OrcFileWriter implements FileWriter { + public static final OrcWriterOptions DEFAULT_OPTION = new OrcWriterOptions(); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - private static final OrcWriterOptions DEFAULT_OPTION = new OrcWriterOptions(); private final OrcWriter orcWriter; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java new file mode 100644 index 0000000000000..5a6ed8c7cc12c --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java @@ -0,0 +1,236 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.orc.FileOrcDataSource; +import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcRecordReader; +import com.facebook.presto.orc.OrcWriter; +import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.raptor.util.Closer; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import io.airlift.log.Logger; +import org.apache.hadoop.fs.FileSystem; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.BitSet; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.IntStream; + +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.OrcPredicate.TRUE; +import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; +import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.raptor.storage.OrcFileWriter.DEFAULT_OPTION; +import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; +import static com.facebook.presto.raptor.storage.OrcStorageManager.HUGE_MAX_READ_BLOCK_SIZE; +import static com.facebook.presto.raptor.util.Closer.closer; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.json.JsonCodec.jsonCodec; +import static io.airlift.units.Duration.nanosSince; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; + +public final class OrcPageFileRewriter + implements FileRewriter +{ + private static final Logger log = Logger.get(OrcPageFileRewriter.class); + private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); + + private final ReaderAttributes readerAttributes; + private final boolean validate; + private final OrcWriterStats stats; + private final TypeManager typeManager; + + OrcPageFileRewriter( + ReaderAttributes readerAttributes, + boolean validate, + OrcWriterStats stats, + TypeManager typeManager) + { + this.readerAttributes = requireNonNull(readerAttributes, "readerAttributes is null"); + this.validate = validate; + this.stats = requireNonNull(stats, "stats is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + public OrcFileInfo rewrite(Map allColumnTypes, File input, File output, BitSet rowsToDelete) + throws IOException + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { + OrcReader reader = new OrcReader( + new FileOrcDataSource(input, readerAttributes.getMaxMergeDistance(), readerAttributes.getMaxReadSize(), readerAttributes.getStreamBufferSize(), readerAttributes.isLazyReadSmallRanges()), + ORC, + readerAttributes.getMaxMergeDistance(), + readerAttributes.getMaxReadSize(), + readerAttributes.getTinyStripeThreshold(), + HUGE_MAX_READ_BLOCK_SIZE); + + if (reader.getFooter().getNumberOfRows() < rowsToDelete.length()) { + throw new IOException("File has fewer rows than deletion vector"); + } + int deleteRowCount = rowsToDelete.cardinality(); + if (reader.getFooter().getNumberOfRows() == deleteRowCount) { + return new OrcFileInfo(0, 0); + } + if (reader.getFooter().getNumberOfRows() >= Integer.MAX_VALUE) { + throw new IOException("File has too many rows"); + } + int inputRowCount = toIntExact(reader.getFooter().getNumberOfRows()); + + Map currentColumnIds = IntStream.range(0, reader.getColumnNames().size()).boxed().collect(toMap(reader.getColumnNames()::get, i -> i)); + + ImmutableList.Builder writerColumnTypesBuilder = ImmutableList.builder(); + ImmutableList.Builder writerColumnIdsBuilder = ImmutableList.builder(); + ImmutableList.Builder readerColumnIndexBuilder = ImmutableList.builder(); + + // Build columns for writer; keep the right ordinal + Map orderedAllColumnTypes = new TreeMap<>(Comparator.comparingLong(Long::parseLong)); + orderedAllColumnTypes.putAll(allColumnTypes); + for (Map.Entry columnType : orderedAllColumnTypes.entrySet()) { + // Get the intersection of the provide columns and the actual columns + Integer currentColumnIndex = currentColumnIds.get(columnType.getKey()); + if (currentColumnIndex != null) { + readerColumnIndexBuilder.add(currentColumnIndex); + writerColumnTypesBuilder.add(columnType.getValue()); + writerColumnIdsBuilder.add(columnType.getKey()); + } + } + + List writerColumnTypes = writerColumnTypesBuilder.build(); + List writerColumnIds = writerColumnIdsBuilder.build(); + List readerColumnIndex = readerColumnIndexBuilder.build(); + Map readerColumns = IntStream.range(0, readerColumnIndex.size()).boxed().collect(toMap(readerColumnIndex::get, writerColumnTypes::get)); + + if (writerColumnTypes.isEmpty()) { + // no intersection; directly return + return new OrcFileInfo(0, 0); + } + + StorageTypeConverter converter = new StorageTypeConverter(typeManager); + List writerStorageTypes = writerColumnTypes.stream() + .map(converter::toStorageType) + .collect(toImmutableList()); + + long start = System.nanoTime(); + + Map userMetadata = ImmutableMap.of(); + if (reader.getFooter().getUserMetadata().containsKey(OrcFileMetadata.KEY)) { + // build metadata if the original file has it + ImmutableMap.Builder metadataBuilder = ImmutableMap.builder(); + for (int i = 0; i < writerColumnIds.size(); i++) { + metadataBuilder.put(Long.parseLong(writerColumnIds.get(i)), writerColumnTypes.get(i).getTypeSignature()); + } + userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(metadataBuilder.build()))); + } + try (Closer recordReader = closer(reader.createRecordReader(readerColumns, TRUE, DEFAULT_STORAGE_TIMEZONE, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE), OrcRecordReader::close); + Closer writer = closer(new OrcWriter( + new OutputStreamOrcDataSink(new FileOutputStream(output)), + writerColumnIds, + writerStorageTypes, + ORC, + SNAPPY, + DEFAULT_OPTION, + userMetadata, + DEFAULT_STORAGE_TIMEZONE, + validate, + HASHED, + stats), + OrcWriter::close)) { + OrcFileInfo fileInfo = rewrite(recordReader.get(), writer.get(), rowsToDelete, writerColumnTypes, readerColumnIndexBuilder.build()); + log.debug("Rewrote file %s in %s (input rows: %s, output rows: %s)", input.getName(), nanosSince(start), inputRowCount, inputRowCount - deleteRowCount); + return fileInfo; + } + } + } + + private static OrcFileInfo rewrite( + OrcRecordReader reader, + OrcWriter writer, + BitSet rowsToDelete, + List types, + List readerColumnIndex) + throws IOException + { + int row = rowsToDelete.nextClearBit(0); + long rowCount = 0; + long uncompressedSize = 0; + + while (true) { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedIOException(); + } + + int batchSize = reader.nextBatch(); + if (batchSize <= 0) { + break; + } + + if (row >= (reader.getFilePosition() + batchSize)) { + continue; + } + + Block[] blocks = new Block[types.size()]; + for (int i = 0; i < types.size(); i++) { + // read from existing columns + blocks[i] = reader.readBlock(types.get(i), readerColumnIndex.get(i)); + } + + row = toIntExact(reader.getFilePosition()); + + Page page = maskedPage(blocks, rowsToDelete, row, batchSize); + writer.write(page); + + rowCount += page.getPositionCount(); + uncompressedSize += page.getLogicalSizeInBytes(); + + row = rowsToDelete.nextClearBit(row + batchSize); + } + + return new OrcFileInfo(rowCount, uncompressedSize); + } + + private static Page maskedPage(Block[] blocks, BitSet rowsToDelete, int start, int count) + { + int[] ids = new int[count]; + int size = 0; + for (int i = 0; i < count; i++) { + if (!rowsToDelete.get(start + i)) { + ids[size] = i; + size++; + } + } + + return new Page(blocks).getPositions(ids, 0, size); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java index 5c70eaf820c40..cb0c62cb99ec3 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java @@ -16,6 +16,7 @@ import com.facebook.presto.raptor.util.Closer; import com.facebook.presto.raptor.util.SyncingFileSystem; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.type.Type; import io.airlift.log.Logger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -68,7 +69,7 @@ public final class OrcRecordFileRewriter private static final Configuration CONFIGURATION = new Configuration(); @Override - public OrcFileInfo rewrite(File input, File output, BitSet rowsToDelete) + public OrcFileInfo rewrite(Map columns, File input, File output, BitSet rowsToDelete) throws IOException { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index e07cb749a71b7..c2c443299057b 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -137,11 +137,11 @@ public class OrcStorageManager // In order to be consistent, we still enforce the conversion. // The following DEFAULT_STORAGE_TIMEZONE is not used by the optimized ORC read/writer given we never read/write timestamp types. public static final DateTimeZone DEFAULT_STORAGE_TIMEZONE = UTC; + // TODO: do not limit the max size of blocks to read for now; enable the limit when the Hive connector is ready + public static final DataSize HUGE_MAX_READ_BLOCK_SIZE = new DataSize(1, PETABYTE); private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); private static final long MAX_ROWS = 1_000_000_000; - // TODO: do not limit the max size of blocks to read for now; enable the limit when the Hive connector is ready - private static final DataSize HUGE_MAX_READ_BLOCK_SIZE = new DataSize(1, PETABYTE); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); private final String nodeId; @@ -228,8 +228,7 @@ public OrcStorageManager( this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); this.orcOptimizedWriterStage = requireNonNull(orcOptimizedWriterStage, "orcOptimizedWriterStage is null"); if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { - // TODO: add new rewriter. - this.fileRewriter = null; + this.fileRewriter = new OrcPageFileRewriter(readerAttributes, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager); } else { this.fileRewriter = new OrcRecordFileRewriter(); @@ -251,7 +250,8 @@ public ConnectorPageSource getPageSource( List columnTypes, TupleDomain effectivePredicate, ReaderAttributes readerAttributes, - OptionalLong transactionId) + OptionalLong transactionId, + Optional> allColumnTypes) { OrcDataSource dataSource = openShard(shardUuid, readerAttributes); @@ -286,7 +286,8 @@ public ConnectorPageSource getPageSource( Optional shardRewriter = Optional.empty(); if (transactionId.isPresent()) { - shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid)); + checkState(allColumnTypes.isPresent()); + shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid, allColumnTypes.get())); } return new OrcPageSource(shardRewriter, recordReader, dataSource, columnIds, columnTypes, columnIndexes.build(), shardUuid, bucketNumber, systemMemoryUsage); @@ -324,13 +325,13 @@ public StoragePageSink createStoragePageSink(long transactionId, OptionalInt buc return new OrcStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); } - private ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid) + private ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns) { return rowsToDelete -> { if (rowsToDelete.isEmpty()) { return completedFuture(ImmutableList.of()); } - return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, rowsToDelete), deletionExecutor); + return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, columns, rowsToDelete), deletionExecutor); }; } @@ -414,7 +415,7 @@ private List computeShardStats(File file) } @VisibleForTesting - Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) + Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns, BitSet rowsToDelete) { if (rowsToDelete.isEmpty()) { return ImmutableList.of(); @@ -424,7 +425,7 @@ Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUI File input = storageService.getStorageFile(shardUuid); File output = storageService.getStagingFile(newShardUuid); - OrcFileInfo info = rewriteFile(input, output, rowsToDelete); + OrcFileInfo info = rewriteFile(columns, input, output, rowsToDelete); long rowCount = info.getRowCount(); if (rowCount == 0) { @@ -453,10 +454,10 @@ private static Collection shardDelta(UUID oldShardUuid, Optional columns, File input, File output, BitSet rowsToDelete) { try { - return fileRewriter.rewrite(input, output, rowsToDelete); + return fileRewriter.rewrite(columns, input, output, rowsToDelete); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to rewrite shard file: " + input, e); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java index 9c938f4dc9713..47e5bd981e3be 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java @@ -19,6 +19,8 @@ import com.facebook.presto.spi.type.Type; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.UUID; @@ -33,7 +35,7 @@ default ConnectorPageSource getPageSource( TupleDomain effectivePredicate, ReaderAttributes readerAttributes) { - return getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, effectivePredicate, readerAttributes, OptionalLong.empty()); + return getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, effectivePredicate, readerAttributes, OptionalLong.empty(), Optional.empty()); } ConnectorPageSource getPageSource( @@ -43,7 +45,8 @@ ConnectorPageSource getPageSource( List columnTypes, TupleDomain effectivePredicate, ReaderAttributes readerAttributes, - OptionalLong transactionId); + OptionalLong transactionId, + Optional> allColumnTypes); StoragePageSink createStoragePageSink( long transactionId, diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java index 50f7d2c4c5bab..83b39ab8d1a43 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.FileOrcDataSource; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcPredicate; @@ -20,6 +22,7 @@ import com.facebook.presto.orc.OrcRecordReader; import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; @@ -102,7 +105,9 @@ public static byte octet(int b) public static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean useOptimizedOrcWriter) { if (useOptimizedOrcWriter) { - return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), new TypeRegistry()); + TypeRegistry typeManager = new TypeRegistry(); + new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), typeManager); } return new OrcRecordWriter(columnIds, columnTypes, file, true); } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java index 60398b6c8c70c..dace510727c97 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java @@ -16,10 +16,14 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcRecordReader; import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.raptor.metadata.TableColumn; +import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.StandardTypes; @@ -31,7 +35,11 @@ import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import io.airlift.json.JsonCodec; +import io.airlift.units.DataSize; +import org.skife.jdbi.v2.DBI; +import org.skife.jdbi.v2.Handle; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -41,13 +49,22 @@ import java.math.BigDecimal; import java.util.BitSet; import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.raptor.storage.FileStorageService.getFileSystemPath; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; +import static com.facebook.presto.raptor.storage.TestOrcStorageManager.createOrcStorageManager; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.tests.StructuralTestUtil.arrayBlockOf; @@ -59,15 +76,19 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.testing.Assertions.assertBetweenInclusive; +import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.nio.file.Files.readAllBytes; import static java.util.UUID.randomUUID; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Test(singleThreaded = true) public class TestOrcFileRewriter { + private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); private File temporary; @@ -204,9 +225,9 @@ public void testRewrite(boolean useOptimizedOrcWriter) rowsToDelete.set(4); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = new OrcRecordFileRewriter().rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 2); - assertEquals(info.getUncompressedSize(), 94); + assertBetweenInclusive(info.getUncompressedSize(), 94L, 94L * 2); try (OrcDataSource dataSource = fileOrcDataSource(newFile)) { OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); @@ -319,9 +340,9 @@ public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) rowsToDelete.set(1); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = new OrcRecordFileRewriter().rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 1); - assertEquals(info.getUncompressedSize(), 13); + assertBetweenInclusive(info.getUncompressedSize(), 13L, 13L * 2); try (OrcDataSource dataSource = fileOrcDataSource(newFile)) { OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); @@ -363,7 +384,7 @@ public void testRewriteAllRowsDeleted(boolean useOptimizedOrcWriter) rowsToDelete.set(1); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = new OrcRecordFileRewriter().rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 0); assertEquals(info.getUncompressedSize(), 0); @@ -385,14 +406,10 @@ public void testRewriteNoRowsDeleted(boolean useOptimizedOrcWriter) BitSet rowsToDelete = new BitSet(); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = new OrcRecordFileRewriter().rewrite(file, newFile, rowsToDelete); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); assertEquals(info.getRowCount(), 2); - assertEquals(info.getUncompressedSize(), 16); - - // TODO: enable check for optimized ORC writer once the rewriter uses the optimized ORC writer as well - if (!useOptimizedOrcWriter) { - assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath())); - } + assertBetweenInclusive(info.getUncompressedSize(), 16L, 16L * 2); + assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath())); } @Test(dataProvider = "useOptimizedOrcWriter") @@ -413,9 +430,258 @@ public void testUncompressedSize(boolean useOptimizedOrcWriter) } File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = new OrcRecordFileRewriter().rewrite(file, newFile, new BitSet()); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, new BitSet()); assertEquals(info.getRowCount(), 3); - assertEquals(info.getUncompressedSize(), 55); + assertBetweenInclusive(info.getUncompressedSize(), 55L, 55L * 2); + } + + /** + * The following test add or drop different columns + */ + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWriter) + throws Exception + { + TypeRegistry typeRegistry = new TypeRegistry(); + DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dbi.registerMapper(new TableColumn.Mapper(typeRegistry)); + Handle dummyHandle = dbi.open(); + File dataDir = Files.createTempDir(); + + StorageManager storageManager = createOrcStorageManager(dbi, dataDir, useOptimizedOrcWriter); + + List columnIds = ImmutableList.of(3L, 7L); + List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); + + File file = new File(temporary, randomUUID().toString()); + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + List pages = rowPagesBuilder(columnTypes) + .row(1L, "1") + .row(2L, "2") + .row(3L, "3") + .row(4L, "4") + .build(); + writer.appendPages(pages); + } + + // Add a column + File newFile1 = new File(temporary, randomUUID().toString()); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), + file, + newFile1, + new BitSet(5)); + assertEquals(info.getRowCount(), 4); + assertEquals(readAllBytes(file.toPath()), readAllBytes(newFile1.toPath())); + + // Drop a column + File newFile2 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), + newFile1, + newFile2, + new BitSet(5)); + assertEquals(info.getRowCount(), 4); + if (useOptimizedOrcWriter) { + // Optimized writer will keep the only column + OrcReader orcReader = new OrcReader(fileOrcDataSource(newFile2), ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + orcReader.getColumnNames().equals(ImmutableList.of("7")); + } + else { + // Apache writer does not drop columns + assertEquals(readAllBytes(newFile1.toPath()), readAllBytes(newFile2.toPath())); + } + + // Add a column with the different ID with different type + File newFile3 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L, 13L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), + newFile2, + newFile3, + new BitSet(5)); + assertEquals(info.getRowCount(), 4); + assertEquals(readAllBytes(newFile2.toPath()), readAllBytes(newFile3.toPath())); + + // Get prepared for the final file; make sure it is accessible from storage manager + UUID uuid = randomUUID(); + File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid); + if (useOptimizedOrcWriter) { + // Apache ORC writer creates the file itself + newFile4.getParentFile().mkdirs(); + newFile4.createNewFile(); + } + + // Drop a column and add a column; also delete 3 rows + BitSet rowsToDelete = new BitSet(5); + rowsToDelete.set(0); + rowsToDelete.set(1); + rowsToDelete.set(3); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 13L, 18L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), + newFile3, + newFile4, + rowsToDelete); + assertEquals(info.getRowCount(), 1); + + ConnectorPageSource source = storageManager.getPageSource( + uuid, + OptionalInt.empty(), + ImmutableList.of(13L, 7L, 18L), + ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), + TupleDomain.all(), + READER_ATTRIBUTES); + + Page page = null; + while (page == null) { + page = source.getNextPage(); + } + assertEquals(page.getPositionCount(), 1); + + // Column 13L + Block column0 = page.getBlock(0); + assertTrue(column0.isNull(0)); + + // Column 7L + Block column1 = page.getBlock(1); + assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("3")); + + // Column 8L + Block column2 = page.getBlock(2); + assertTrue(column2.isNull(0)); + + // Remove all the columns + File newFile5 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(13L, 18L), ImmutableList.of(createVarcharType(5), INTEGER)), + newFile4, + newFile5, + new BitSet(5)); + if (useOptimizedOrcWriter) { + // Optimized writer will drop the file + assertEquals(info.getRowCount(), 0); + assertFalse(newFile5.exists()); + } + else { + assertEquals(info.getRowCount(), 1); + assertEquals(readAllBytes(newFile4.toPath()), readAllBytes(newFile5.toPath())); + } + + dummyHandle.close(); + deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); + } + + /** + * The following test drop and add the same columns; the legacy ORC rewriter will fail due to unchanged schema. + * However, if we enforce the newly added column to always have the largest ID, this won't happen. + */ + @Test(dataProvider = "useOptimizedOrcWriter") + public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) + throws Exception + { + TypeRegistry typeRegistry = new TypeRegistry(); + DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dbi.registerMapper(new TableColumn.Mapper(typeRegistry)); + Handle dummyHandle = dbi.open(); + File dataDir = Files.createTempDir(); + + StorageManager storageManager = createOrcStorageManager(dbi, dataDir, useOptimizedOrcWriter); + + List columnIds = ImmutableList.of(3L, 7L); + List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); + + File file = new File(temporary, randomUUID().toString()); + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + List pages = rowPagesBuilder(columnTypes) + .row(2L, "2") + .build(); + writer.appendPages(pages); + } + + // Add a column + File newFile1 = new File(temporary, randomUUID().toString()); + OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), + file, + newFile1, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + // Drop a column + File newFile2 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), + newFile1, + newFile2, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + // Add a column with the same ID but different type + File newFile3 = new File(temporary, randomUUID().toString()); + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 10L, 3L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), + newFile2, + newFile3, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + // Get prepared for the final file; make sure it is accessible from storage manager + UUID uuid = randomUUID(); + File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid); + if (useOptimizedOrcWriter) { + // Apache ORC writer creates the file itself + newFile4.getParentFile().mkdirs(); + newFile4.createNewFile(); + } + + // Drop a column and add a column + info = createFileRewriter(useOptimizedOrcWriter).rewrite( + getColumnTypes(ImmutableList.of(7L, 3L, 8L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), + newFile3, + newFile4, + new BitSet(5)); + assertEquals(info.getRowCount(), 1); + + ConnectorPageSource source = storageManager.getPageSource( + uuid, + OptionalInt.empty(), + ImmutableList.of(3L, 7L, 8L), + ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), + TupleDomain.all(), + READER_ATTRIBUTES); + + Page page = null; + while (page == null) { + page = source.getNextPage(); + } + assertEquals(page.getPositionCount(), 1); + + try { + // Column 3L + Block column0 = page.getBlock(0); + assertTrue(column0.isNull(0)); + + // Column 7L + Block column1 = page.getBlock(1); + assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("2")); + + // Column 8L + Block column2 = page.getBlock(2); + assertTrue(column2.isNull(0)); + + dummyHandle.close(); + deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); + + if (!useOptimizedOrcWriter) { + // Apache ORC rewriter will not respect the schema + fail(); + } + } + catch (UnsupportedOperationException e) { + if (useOptimizedOrcWriter) { + // Optimized ORC rewriter will respect the schema + fail(); + } + } } private static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean writeMetadata, boolean useOptimizedOrcWriter) @@ -425,4 +691,19 @@ private static FileWriter createFileWriter(List columnIds, List colu } return new OrcRecordWriter(columnIds, columnTypes, file, writeMetadata); } + + private static FileRewriter createFileRewriter(boolean useOptimizedOrcWriter) + { + if (useOptimizedOrcWriter) { + TypeRegistry typeManager = new TypeRegistry(); + new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + return new OrcPageFileRewriter(READER_ATTRIBUTES, true, new OrcWriterStats(), typeManager); + } + return new OrcRecordFileRewriter(); + } + + private static Map getColumnTypes(List columnIds, List columnTypes) + { + return IntStream.range(0, columnIds.size()).boxed().collect(Collectors.toMap(index -> String.valueOf(columnIds.get(index)), columnTypes::get)); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java index 6f96e0a04bcc4..622b7a4e1e715 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java @@ -67,6 +67,8 @@ import java.util.OptionalLong; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; @@ -347,7 +349,12 @@ public void testRewriter(boolean useOptimizedOrcWriter) // delete one row BitSet rowsToDelete = new BitSet(); rowsToDelete.set(0); - Collection fragments = manager.rewriteShard(transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete); + Collection fragments = manager.rewriteShard( + transactionId, + OptionalInt.empty(), + shards.get(0).getShardUuid(), + IntStream.range(0, columnIds.size()).boxed().collect(Collectors.toMap(index -> String.valueOf(columnIds.get(index)), columnTypes::get)), + rowsToDelete); Slice shardDelta = Iterables.getOnlyElement(fragments); ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes()); From 5b3c7d0cc986894a996d98080f6ccee8ef5eef92 Mon Sep 17 00:00:00 2001 From: James Sun Date: Sun, 7 Apr 2019 13:36:34 -0700 Subject: [PATCH 26/66] Add optimized writer integration tests --- .../TestRaptorDistributedQueries.java | 4 +- .../TestRaptorDistributedQueriesBucketed.java | 4 +- .../TestRaptorIntegrationSmokeTest.java | 4 +- ...estRaptorIntegrationSmokeTestBucketed.java | 4 +- ...nSmokeTestBucketedWithOptimizedWriter.java | 39 +++++++++++++++++++ .../TestRaptorIntegrationSmokeTestMySql.java | 3 +- ...tegrationSmokeTestWithOptimizedWriter.java | 33 ++++++++++++++++ 7 files changed, 82 insertions(+), 9 deletions(-) rename presto-raptor/src/test/java/com/facebook/presto/raptor/{ => integration}/TestRaptorDistributedQueries.java (89%) rename presto-raptor/src/test/java/com/facebook/presto/raptor/{ => integration}/TestRaptorDistributedQueriesBucketed.java (87%) rename presto-raptor/src/test/java/com/facebook/presto/raptor/{ => integration}/TestRaptorIntegrationSmokeTest.java (99%) rename presto-raptor/src/test/java/com/facebook/presto/raptor/{ => integration}/TestRaptorIntegrationSmokeTestBucketed.java (91%) create mode 100644 presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java rename presto-raptor/src/test/java/com/facebook/presto/raptor/{ => integration}/TestRaptorIntegrationSmokeTestMySql.java (96%) create mode 100644 presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueries.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java similarity index 89% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueries.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java index 39fe2ed023932..62467a93d98fe 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueries.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; @@ -24,7 +24,7 @@ public class TestRaptorDistributedQueries @SuppressWarnings("unused") public TestRaptorDistributedQueries() { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false)); + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } protected TestRaptorDistributedQueries(QueryRunnerSupplier supplier) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueriesBucketed.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java similarity index 87% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueriesBucketed.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java index cd7aecf3a7435..e1dec16bc3a53 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorDistributedQueriesBucketed.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.google.common.collect.ImmutableMap; @@ -22,7 +22,7 @@ public class TestRaptorDistributedQueriesBucketed { public TestRaptorDistributedQueriesBucketed() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true)); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } @Override diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTest.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java similarity index 99% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTest.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java index 9378f647ff9d0..a94d80e22deac 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTest.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.testing.MaterializedResult; @@ -60,7 +60,7 @@ public class TestRaptorIntegrationSmokeTest @SuppressWarnings("unused") public TestRaptorIntegrationSmokeTest() { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false)); + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "DISABLED"))); } protected TestRaptorIntegrationSmokeTest(QueryRunnerSupplier supplier) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestBucketed.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java similarity index 91% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestBucketed.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java index 88716b37d481d..ef58207d9dc73 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestBucketed.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -23,7 +23,7 @@ public class TestRaptorIntegrationSmokeTestBucketed { public TestRaptorIntegrationSmokeTestBucketed() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true)); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "DISABLED"))); } @Test diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java new file mode 100644 index 0000000000000..d2b1b8597fd3e --- /dev/null +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.integration; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; + +public class TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter + extends TestRaptorIntegrationSmokeTest +{ + public TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter() + { + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + } + + @Test + public void testShardsSystemTableBucketNumber() + { + assertQuery("" + + "SELECT count(DISTINCT bucket_number)\n" + + "FROM system.shards\n" + + "WHERE table_schema = 'tpch'\n" + + " AND table_name = 'orders'", + "SELECT 25"); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestMySql.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java similarity index 96% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestMySql.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java index bd9671c224a70..595824864e29e 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorIntegrationSmokeTestMySql.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java @@ -11,8 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor; +package com.facebook.presto.raptor.integration; +import com.facebook.presto.raptor.RaptorPlugin; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java new file mode 100644 index 0000000000000..28eff3d093f29 --- /dev/null +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.integration; + +import com.google.common.collect.ImmutableMap; + +import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; + +public class TestRaptorIntegrationSmokeTestWithOptimizedWriter + extends TestRaptorIntegrationSmokeTest +{ + @SuppressWarnings("unused") + public TestRaptorIntegrationSmokeTestWithOptimizedWriter() + { + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + } + + protected TestRaptorIntegrationSmokeTestWithOptimizedWriter(QueryRunnerSupplier supplier) + { + super(supplier); + } +} From 6727da11e22c9682c21c409adda1a3fc4680c903 Mon Sep 17 00:00:00 2001 From: James Sun Date: Thu, 18 Apr 2019 14:16:16 -0700 Subject: [PATCH 27/66] Add testDelete for Raptor smoke tests --- .../TestRaptorIntegrationSmokeTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java index a94d80e22deac..f5ddd50d515ac 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java @@ -771,4 +771,28 @@ public void testAlterTable() assertUpdate("DROP TABLE test_alter_table"); } + + @Test + public void testDelete() + { + assertUpdate("CREATE TABLE test_delete_table (c1 bigint, c2 bigint)"); + assertUpdate("INSERT INTO test_delete_table VALUES (1, 1), (1, 2), (1, 3), (1, 4), (11, 1), (11, 2)", 6); + + assertUpdate("ALTER TABLE test_delete_table ADD COLUMN c3 bigint"); + assertUpdate("INSERT INTO test_delete_table VALUES (2, 1, 1), (2, 2, 2), (2, 3, 3), (2, 4, 4), (22, 1, 1), (22, 2, 2), (22, 4, 4)", 7); + + assertUpdate("DELETE FROM test_delete_table WHERE c1 = 1", 4); + assertQuery("SELECT * FROM test_delete_table", "VALUES (11, 1, NULL), (11, 2, NULL), (2, 1, 1), (2, 2, 2), (2, 3, 3), (2, 4, 4), (22, 1, 1), (22, 2, 2), (22, 4, 4)"); + + assertUpdate("ALTER TABLE test_delete_table DROP COLUMN c2"); + assertUpdate("INSERT INTO test_delete_table VALUES (3, 1), (3, 2), (3, 3), (3, 4)", 4); + + assertUpdate("DELETE FROM test_delete_table WHERE c1 = 2", 4); + assertQuery("SELECT * FROM test_delete_table", "VALUES (11, NULL), (11, NULL), (22, 1), (22, 2), (22, 4), (3, 1), (3, 2), (3, 3), (3, 4)"); + + assertUpdate("DELETE FROM test_delete_table WHERE c1 % 11 = 0", 5); + assertQuery("SELECT * FROM test_delete_table", "VALUES (3, 1), (3, 2), (3, 3), (3, 4)"); + + assertUpdate("DROP TABLE test_delete_table"); + } } From 91dd2a536f4f3e13ec12edf9e3b225156ab543cb Mon Sep 17 00:00:00 2001 From: rongrong Date: Thu, 18 Apr 2019 16:08:19 -0700 Subject: [PATCH 28/66] Move StaticFunctionHandle to presto-main --- .../java/com/facebook/presto/metadata/StaticFunctionHandle.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {presto-spi => presto-main}/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java (100%) diff --git a/presto-spi/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java similarity index 100% rename from presto-spi/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java rename to presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java From 4826bd13986a78ba887c7ed38e4ecdef9c9dbb0f Mon Sep 17 00:00:00 2001 From: James Sun Date: Wed, 17 Apr 2019 18:31:51 -0700 Subject: [PATCH 29/66] Move DefaultRowExpressionTraversalVisitor to SPI DefaultRowExpressionTraversalVisitor could be a generic helper. --- .../facebook/presto/sql/planner/SymbolsExtractor.java | 2 +- .../DefaultRowExpressionTraversalVisitor.java | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) rename {presto-main/src/main/java/com/facebook/presto/sql/relational => presto-spi/src/main/java/com/facebook/presto/spi/relation}/DefaultRowExpressionTraversalVisitor.java (78%) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java index 561c183e5d9c7..91e531b69bb64 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.relation.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.plan.PlanNode; -import com.facebook.presto.sql.relational.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.sql.tree.DefaultExpressionTraversalVisitor; import com.facebook.presto.sql.tree.DefaultTraversalVisitor; import com.facebook.presto.sql.tree.DereferenceExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/DefaultRowExpressionTraversalVisitor.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java similarity index 78% rename from presto-main/src/main/java/com/facebook/presto/sql/relational/DefaultRowExpressionTraversalVisitor.java rename to presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java index 945dc4071840a..5c4325498e621 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/DefaultRowExpressionTraversalVisitor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java @@ -11,15 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.relational; - -import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.ConstantExpression; -import com.facebook.presto.spi.relation.InputReferenceExpression; -import com.facebook.presto.spi.relation.LambdaDefinitionExpression; -import com.facebook.presto.spi.relation.RowExpressionVisitor; -import com.facebook.presto.spi.relation.SpecialFormExpression; -import com.facebook.presto.spi.relation.VariableReferenceExpression; +package com.facebook.presto.spi.relation; /** * The default visitor serves as a template for "consumer-like" tree traversal. From f094084482ae0f907222c7aac1c233f3a6d0d5b0 Mon Sep 17 00:00:00 2001 From: James Sun Date: Mon, 15 Apr 2019 08:20:18 -0700 Subject: [PATCH 30/66] Construct RowExpressionDomainTranslator with metadata --- .../optimizations/PropertyDerivations.java | 2 +- .../RowExpressionDomainTranslator.java | 28 ++++---- .../TestRowExpressionDomainTranslator.java | 65 ++++++++++++------- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java index e3f23a9c06be6..2e7dac132c4b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java @@ -600,7 +600,7 @@ public ActualProperties visitFilter(FilterNode node, List inpu tupleDomain = ExpressionDomainTranslator.fromPredicate(metadata, session, castToExpression(node.getPredicate()), types).getTupleDomain(); } else { - tupleDomain = RowExpressionDomainTranslator.fromPredicate(metadata, session, node.getPredicate()) + tupleDomain = new RowExpressionDomainTranslator(metadata).fromPredicate(session, node.getPredicate()) .getTupleDomain() .transform(column -> new Symbol(column.getName())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index 040c1af2d2ba3..6fd12c9e764b1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -91,10 +91,12 @@ public final class RowExpressionDomainTranslator private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; private final StandardFunctionResolution functionResolution; + private final Metadata metadata; - public RowExpressionDomainTranslator(FunctionManager functionManager) + public RowExpressionDomainTranslator(Metadata metadata) { - this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.functionManager = metadata.getFunctionManager(); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); this.functionResolution = new StandardFunctionResolution(functionManager); } @@ -112,6 +114,17 @@ public RowExpression toPredicate(TupleDomain tupleD .collect(collectingAndThen(toImmutableList(), logicalRowExpressions::combineConjuncts)); } + /** + * Convert an Expression predicate into an ExtractionResult consisting of: + * 1) A successfully extracted TupleDomain + * 2) An Expression fragment which represents the part of the original Expression that will need to be re-evaluated + * after filtering with the TupleDomain. + */ + public ExtractionResult fromPredicate(Session session, RowExpression predicate) + { + return predicate.accept(new Visitor(metadata, session), false); + } + private RowExpression toPredicate(Domain domain, VariableReferenceExpression reference) { if (domain.getValues().isNone()) { @@ -276,17 +289,6 @@ private static boolean isBetween(Range range) && !range.getHigh().isUpperUnbounded() && range.getHigh().getBound() == Marker.Bound.EXACTLY; } - /** - * Convert an Expression predicate into an ExtractionResult consisting of: - * 1) A successfully extracted TupleDomain - * 2) An Expression fragment which represents the part of the original Expression that will need to be re-evaluated - * after filtering with the TupleDomain. - */ - public static ExtractionResult fromPredicate(Metadata metadata, Session session, RowExpression predicate) - { - return predicate.accept(new Visitor(metadata, session), false); - } - private static class Visitor implements RowExpressionVisitor { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index d0ac90767d0a9..798a5ba3862fe 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -128,7 +128,7 @@ public class TestRowExpressionDomainTranslator public void setup() { metadata = createTestMetadataManager(); - domainTranslator = new RowExpressionDomainTranslator(metadata.getFunctionManager()); + domainTranslator = new RowExpressionDomainTranslator(metadata); } @AfterClass(alwaysRun = true) @@ -1074,46 +1074,46 @@ private void testNumericTypeTranslation(NumericValues columnValues, NumericValue @Test public void testLegacyCharComparedToVarcharExpression() { - metadata = createTestMetadataManager(new FeaturesConfig().setLegacyCharToVarcharCoercion(true)); + RowExpressionDomainTranslator domainTranslator = new RowExpressionDomainTranslator(createTestMetadataManager(new FeaturesConfig().setLegacyCharToVarcharCoercion(true))); String maxCodePoint = new String(Character.toChars(Character.MAX_CODE_POINT)); // greater than or equal - testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.greaterThan(createCharType(10), utf8Slice("123456788" + maxCodePoint))); - testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.greaterThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890"))); - testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))); + testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.greaterThan(createCharType(10), utf8Slice("123456788" + maxCodePoint)), domainTranslator); + testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.greaterThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); + testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); // greater than - testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.greaterThan(createCharType(10), utf8Slice("123456788" + maxCodePoint))); - testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))); - testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))); + testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.greaterThan(createCharType(10), utf8Slice("123456788" + maxCodePoint)), domainTranslator); + testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); + testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); // less than or equal - testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.lessThanOrEqual(createCharType(10), utf8Slice("123456788" + maxCodePoint))); - testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890"))); - testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890"))); + testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.lessThanOrEqual(createCharType(10), utf8Slice("123456788" + maxCodePoint)), domainTranslator); + testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); + testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); // less than - testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.lessThanOrEqual(createCharType(10), utf8Slice("123456788" + maxCodePoint))); - testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890"))); - testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890"))); + testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.lessThanOrEqual(createCharType(10), utf8Slice("123456788" + maxCodePoint)), domainTranslator); + testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); + testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); // equal - testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.none(createCharType(10))); - testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.equal(createCharType(10), Slices.utf8Slice("1234567890"))); - testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.none(createCharType(10))); + testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.none(createCharType(10)), domainTranslator); + testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.equal(createCharType(10), Slices.utf8Slice("1234567890")), domainTranslator); + testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.none(createCharType(10)), domainTranslator); // not equal - testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.notNull(createCharType(10))); + testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.notNull(createCharType(10)), domainTranslator); testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Domain.create(ValueSet.ofRanges( - Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))), false)); - testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.notNull(createCharType(10))); + Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))), false), domainTranslator); + testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.notNull(createCharType(10)), domainTranslator); // is distinct from - testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.all(createCharType(10))); + testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.all(createCharType(10)), domainTranslator); testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Domain.create(ValueSet.ofRanges( - Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))), true)); - testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.all(createCharType(10))); + Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))), true), domainTranslator); + testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.all(createCharType(10)), domainTranslator); } @Test @@ -1154,7 +1154,12 @@ private RowExpression toPredicate(TupleDomain tuple private ExtractionResult fromPredicate(RowExpression originalPredicate) { - return RowExpressionDomainTranslator.fromPredicate(metadata, TEST_SESSION, originalPredicate); + return fromPredicate(originalPredicate, domainTranslator); + } + + private ExtractionResult fromPredicate(RowExpression originalPredicate, RowExpressionDomainTranslator domainTranslator) + { + return domainTranslator.fromPredicate(TEST_SESSION, originalPredicate); } private RowExpression nullLiteral(Type type) @@ -1325,9 +1330,19 @@ private void testSimpleComparison(RowExpression expression, VariableReferenceExp testSimpleComparison(expression, input, Domain.create(ValueSet.ofRanges(expectedDomainRange), false)); } + private void testSimpleComparison(RowExpression expression, VariableReferenceExpression input, Range expectedDomainRange, RowExpressionDomainTranslator domainTranslator) + { + testSimpleComparison(expression, input, Domain.create(ValueSet.ofRanges(expectedDomainRange), false), domainTranslator); + } + private void testSimpleComparison(RowExpression expression, VariableReferenceExpression input, Domain domain) { - ExtractionResult result = fromPredicate(expression); + testSimpleComparison(expression, input, domain, domainTranslator); + } + + private void testSimpleComparison(RowExpression expression, VariableReferenceExpression input, Domain domain, RowExpressionDomainTranslator domainTranslator) + { + ExtractionResult result = fromPredicate(expression, domainTranslator); assertEquals(result.getRemainingExpression(), TRUE); TupleDomain actual = result.getTupleDomain(); TupleDomain expected = withColumnDomains(ImmutableMap.of(input, domain)); From 43f627f61f576a865ae932fae9d6b5caf23f947e Mon Sep 17 00:00:00 2001 From: James Sun Date: Mon, 15 Apr 2019 08:30:58 -0700 Subject: [PATCH 31/66] Use connector session in RowExpression utility classes Replace Session with ConnectorSession in RowExpressionInterpreter and RowExpressionDomainTranslator. --- .../presto/cost/FilterStatsCalculator.java | 2 +- .../presto/cost/ScalarStatsCalculator.java | 2 +- .../facebook/presto/cost/ValuesStatsRule.java | 2 +- .../sql/planner/LocalExecutionPlanner.java | 4 ++-- .../sql/planner/RowExpressionInterpreter.java | 22 +++++++++---------- .../optimizations/PropertyDerivations.java | 2 +- .../RowExpressionDomainTranslator.java | 12 +++++----- .../presto/sql/TestExpressionInterpreter.java | 4 ++-- .../TestRowExpressionDomainTranslator.java | 2 +- .../assertions/RowExpressionVerifier.java | 2 +- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index bb94750dc20a6..091ca8687f2ba 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -151,7 +151,7 @@ private Expression simplifyExpression(Session session, Expression predicate, Typ private RowExpression simplifyExpression(Session session, RowExpression predicate) { - RowExpressionInterpreter interpreter = new RowExpressionInterpreter(predicate, metadata, session, true); + RowExpressionInterpreter interpreter = new RowExpressionInterpreter(predicate, metadata, session.toConnectorSession(), true); Object value = interpreter.optimize(); if (value == null) { diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 28d5ae2826eb5..44d0f17f5edd5 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -122,7 +122,7 @@ public SymbolStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - Object value = new RowExpressionInterpreter(call, metadata, session, true).optimize(); + Object value = new RowExpressionInterpreter(call, metadata, session.toConnectorSession(), true).optimize(); if (value == null) { return nullStatsEstimate(); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java index b752be0e0ba8c..7da57cc4438aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java @@ -88,7 +88,7 @@ private List getSymbolValues(ValuesNode valuesNode, int symbolId, Sessio if (isExpression(rowExpression)) { return evaluateConstantExpression(castToExpression(rowExpression), symbolType, metadata, session, ImmutableList.of()); } - return evaluateConstantRowExpression(rowExpression, metadata, session); + return evaluateConstantRowExpression(rowExpression, metadata, session.toConnectorSession()); }) .collect(toList()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index a48143df2a70c..175a9ac55e327 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -1206,7 +1206,7 @@ private PhysicalOperation visitScanFilterAndProject( // filterExpression may contain large function calls; evaluate them before compiling. if (filterExpression.isPresent()) { Type type = filterExpression.get().getType(); - Object value = new RowExpressionInterpreter(filterExpression.get(), metadata, session, true).optimize(); + Object value = new RowExpressionInterpreter(filterExpression.get(), metadata, session.toConnectorSession(), true).optimize(); if (value instanceof RowExpression) { RowExpression optimizedFilter = (RowExpression) value; // building channel info @@ -1320,7 +1320,7 @@ public PhysicalOperation visitValues(ValuesNode node, LocalExecutionPlanContext pageBuilder.declarePosition(); for (int i = 0; i < row.size(); i++) { // evaluate the literal value - Object result = rowExpressionInterpreter(row.get(i), metadata, context.getSession()).evaluate(); + Object result = rowExpressionInterpreter(row.get(i), metadata, context.getSession().toConnectorSession()).evaluate(); writeNativeValue(outputTypes.get(i), pageBuilder.getBlockBuilder(i), result); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 8f63e672411e6..e42fb9c789c33 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner; -import com.facebook.presto.Session; import com.facebook.presto.client.FailureInfo; import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RowBlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; @@ -102,7 +102,7 @@ public class RowExpressionInterpreter { private final RowExpression expression; private final Metadata metadata; - private final Session session; + private final ConnectorSession session; private final boolean optimize; private final InterpretedFunctionInvoker functionInvoker; private final com.facebook.presto.sql.relational.DeterminismEvaluator determinismEvaluator; @@ -110,7 +110,7 @@ public class RowExpressionInterpreter private final Visitor visitor; - public static Object evaluateConstantRowExpression(RowExpression expression, Metadata metadata, Session session) + public static Object evaluateConstantRowExpression(RowExpression expression, Metadata metadata, ConnectorSession session) { // evaluate the expression Object result = new RowExpressionInterpreter(expression, metadata, session, false).evaluate(); @@ -118,12 +118,12 @@ public static Object evaluateConstantRowExpression(RowExpression expression, Met return result; } - public static RowExpressionInterpreter rowExpressionInterpreter(RowExpression expression, Metadata metadata, Session session) + public static RowExpressionInterpreter rowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session) { return new RowExpressionInterpreter(expression, metadata, session, false); } - public RowExpressionInterpreter(RowExpression expression, Metadata metadata, Session session, boolean optimize) + public RowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session, boolean optimize) { this.expression = requireNonNull(expression, "expression is null"); this.metadata = requireNonNull(metadata, "metadata is null"); @@ -236,7 +236,7 @@ public Object visitCall(CallExpression node, Object context) if (optimize && (!functionMetadata.isDeterministic() || hasUnresolvedValue(argumentValues) || functionMetadata.getName().equals("fail"))) { return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, argumentTypes)); } - return functionInvoker.invoke(functionHandle, session.toConnectorSession(), argumentValues); + return functionInvoker.invoke(functionHandle, session, argumentValues); } @Override @@ -316,8 +316,8 @@ else if (Boolean.TRUE.equals(condition)) { EQUAL, ImmutableList.of(commonType, commonType), ImmutableList.of( - functionInvoker.invoke(firstCast, session.toConnectorSession(), left), - functionInvoker.invoke(secondCast, session.toConnectorSession(), right)))); + functionInvoker.invoke(firstCast, session, left), + functionInvoker.invoke(secondCast, session, right)))); if (equal) { return null; @@ -633,7 +633,7 @@ private RowExpression createFailureFunction(RuntimeException exception, Type typ String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); FunctionHandle jsonParse = metadata.getFunctionManager().lookupFunction("json_parse", fromTypes(VARCHAR)); - Object json = functionInvoker.invoke(jsonParse, session.toConnectorSession(), utf8Slice(failureInfo)); + Object json = functionInvoker.invoke(jsonParse, session, utf8Slice(failureInfo)); FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction("fail", fromTypes(JSON)); FunctionHandle cast = metadata.getFunctionManager().lookupCast(CAST, UNKNOWN.getTypeSignature(), type.getTypeSignature()); @@ -654,7 +654,7 @@ private boolean hasUnresolvedValue(List values) private Object invokeOperator(OperatorType operatorType, List argumentTypes, List argumentValues) { FunctionHandle operatorHandle = metadata.getFunctionManager().resolveOperator(operatorType, fromTypes(argumentTypes)); - return functionInvoker.invoke(operatorHandle, session.toConnectorSession(), argumentValues); + return functionInvoker.invoke(operatorHandle, session, argumentValues); } private List toRowExpressions(List values, List types) @@ -762,7 +762,7 @@ private SpecialCallResult tryHandleLike(CallExpression callExpression, List inpu tupleDomain = ExpressionDomainTranslator.fromPredicate(metadata, session, castToExpression(node.getPredicate()), types).getTupleDomain(); } else { - tupleDomain = new RowExpressionDomainTranslator(metadata).fromPredicate(session, node.getPredicate()) + tupleDomain = new RowExpressionDomainTranslator(metadata).fromPredicate(session.toConnectorSession(), node.getPredicate()) .getTupleDomain() .transform(column -> new Symbol(column.getName())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index 6fd12c9e764b1..48b2bbf8a89f3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.sql.relational; -import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.OperatorNotFoundException; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; @@ -120,7 +120,7 @@ public RowExpression toPredicate(TupleDomain tupleD * 2) An Expression fragment which represents the part of the original Expression that will need to be re-evaluated * after filtering with the TupleDomain. */ - public ExtractionResult fromPredicate(Session session, RowExpression predicate) + public ExtractionResult fromPredicate(ConnectorSession session, RowExpression predicate) { return predicate.accept(new Visitor(metadata, session), false); } @@ -294,13 +294,13 @@ private static class Visitor { private final InterpretedFunctionInvoker functionInvoker; private final Metadata metadata; - private final Session session; + private final ConnectorSession session; private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; private final DeterminismEvaluator determinismEvaluator; private final StandardFunctionResolution resolution; - private Visitor(Metadata metadata, Session session) + private Visitor(Metadata metadata, ConnectorSession session) { this.functionInvoker = new InterpretedFunctionInvoker(metadata.getFunctionManager()); this.metadata = metadata; @@ -549,7 +549,7 @@ private RowExpression binaryOperator(OperatorType operatorType, RowExpression le private Optional floorValue(Type fromType, Type toType, Object value) { return getSaturatedFloorCastOperator(fromType, toType) - .map((operator) -> functionInvoker.invoke(operator, session.toConnectorSession(), value)); + .map((operator) -> functionInvoker.invoke(operator, session, value)); } private Optional getSaturatedFloorCastOperator(Type fromType, Type toType) @@ -565,7 +565,7 @@ private Optional getSaturatedFloorCastOperator(Type fromType, Ty private int compareOriginalValueToCoerced(Type originalValueType, Object originalValue, Type coercedValueType, Object coercedValue) { FunctionHandle castToOriginalTypeOperator = metadata.getFunctionManager().lookupCast(CAST, coercedValueType.getTypeSignature(), originalValueType.getTypeSignature()); - Object coercedValueInOriginalType = functionInvoker.invoke(castToOriginalTypeOperator, session.toConnectorSession(), coercedValue); + Object coercedValueInOriginalType = functionInvoker.invoke(castToOriginalTypeOperator, session, coercedValue); Block originalValueBlock = Utils.nativeValueToBlock(originalValueType, originalValue); Block coercedValueBlock = Utils.nativeValueToBlock(originalValueType, coercedValueInOriginalType); return originalValueType.compareTo(originalValueBlock, 0, coercedValueBlock, 0); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index fba62a6b1f0e4..730514c98d6a3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -1530,7 +1530,7 @@ private static Object optimize(@Language("SQL") String expression) return value; }); RowExpression rowExpression = TRANSLATOR.translate(parsedExpression, SYMBOL_TYPES); - Object rowExpressionResult = new RowExpressionInterpreter(rowExpression, METADATA, TEST_SESSION, true).optimize(symbol -> { + Object rowExpressionResult = new RowExpressionInterpreter(rowExpression, METADATA, TEST_SESSION.toConnectorSession(), true).optimize(symbol -> { Object value = symbolConstant(symbol); if (value == null) { return new VariableReferenceExpression(symbol.getName(), SYMBOL_TYPES.get(symbol)); @@ -1675,7 +1675,7 @@ private static Object evaluate(Expression expression, boolean deterministic) { Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, SYMBOL_TYPES, expression, emptyList(), WarningCollector.NOOP); Object expressionResult = expressionInterpreter(expression, METADATA, TEST_SESSION, expressionTypes).evaluate(); - Object rowExpressionResult = rowExpressionInterpreter(TRANSLATOR.translateAndOptimize(expression), METADATA, TEST_SESSION).evaluate(); + Object rowExpressionResult = rowExpressionInterpreter(TRANSLATOR.translateAndOptimize(expression), METADATA, TEST_SESSION.toConnectorSession()).evaluate(); if (deterministic) { assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index 798a5ba3862fe..1b8e6b24bd866 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -1159,7 +1159,7 @@ private ExtractionResult fromPredicate(RowExpression originalPredicate) private ExtractionResult fromPredicate(RowExpression originalPredicate, RowExpressionDomainTranslator domainTranslator) { - return domainTranslator.fromPredicate(TEST_SESSION, originalPredicate); + return domainTranslator.fromPredicate(TEST_SESSION.toConnectorSession(), originalPredicate); } private RowExpression nullLiteral(Type type) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index 9d3353c61a0d6..a7382bf40fc57 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -343,7 +343,7 @@ else if (expression instanceof GenericLiteral) { private Boolean compareLiteral(Node expected, RowExpression actual) { if (actual instanceof CallExpression && functionResolution.isCastFunction(((CallExpression) actual).getFunctionHandle())) { - return getValueFromLiteral(expected).equals(String.valueOf(rowExpressionInterpreter(actual, metadata, session).evaluate())); + return getValueFromLiteral(expected).equals(String.valueOf(rowExpressionInterpreter(actual, metadata, session.toConnectorSession()).evaluate())); } if (actual instanceof ConstantExpression) { return getValueFromLiteral(expected).equals(String.valueOf(LiteralInterpreter.evaluate(TEST_SESSION.toConnectorSession(), (ConstantExpression) actual))); From c6046e5b7334c5075191a506875f8bca76d1b7fb Mon Sep 17 00:00:00 2001 From: James Sun Date: Mon, 15 Apr 2019 08:53:47 -0700 Subject: [PATCH 32/66] Abstract DomainTranslator in SPI --- .../planner/ExpressionDomainTranslator.java | 1 + .../RowExpressionDomainTranslator.java | 36 +++---------- .../TestRowExpressionDomainTranslator.java | 2 +- .../presto/spi/relation/DomainTranslator.java | 54 +++++++++++++++++++ 4 files changed, 62 insertions(+), 31 deletions(-) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java index e6ef335a0ea98..1a2db01207f96 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java @@ -85,6 +85,7 @@ import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.toList; +// No need to inherit from DomainTranslator; the class will be deprecated in the long term @Deprecated public final class ExpressionDomainTranslator { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index 48b2bbf8a89f3..8a9955f3474f5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -33,6 +33,7 @@ import com.facebook.presto.spi.predicate.ValueSet; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -87,6 +88,7 @@ import static java.util.stream.Collectors.toList; public final class RowExpressionDomainTranslator + implements DomainTranslator { private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; @@ -101,6 +103,7 @@ public RowExpressionDomainTranslator(Metadata metadata) this.functionResolution = new StandardFunctionResolution(functionManager); } + @Override public RowExpression toPredicate(TupleDomain tupleDomain) { if (tupleDomain.isNone()) { @@ -114,12 +117,7 @@ public RowExpression toPredicate(TupleDomain tupleD .collect(collectingAndThen(toImmutableList(), logicalRowExpressions::combineConjuncts)); } - /** - * Convert an Expression predicate into an ExtractionResult consisting of: - * 1) A successfully extracted TupleDomain - * 2) An Expression fragment which represents the part of the original Expression that will need to be re-evaluated - * after filtering with the TupleDomain. - */ + @Override public ExtractionResult fromPredicate(ConnectorSession session, RowExpression predicate) { return predicate.accept(new Visitor(metadata, session), false); @@ -331,12 +329,12 @@ public ExtractionResult visitSpecialForm(SpecialFormExpression node, Boolean com ExtractionResult extractionResult = or(disjuncts.build()).accept(this, complement); // preserve original IN predicate as remaining predicate - if (extractionResult.tupleDomain.isAll()) { + if (extractionResult.getTupleDomain().isAll()) { RowExpression originalPredicate = node; if (complement) { originalPredicate = not(resolution, originalPredicate); } - return new ExtractionResult(extractionResult.tupleDomain, originalPredicate); + return new ExtractionResult(extractionResult.getTupleDomain(), originalPredicate); } return extractionResult; } @@ -882,26 +880,4 @@ public NullableValue getValue() return value; } } - - public static class ExtractionResult - { - private final TupleDomain tupleDomain; - private final RowExpression remainingExpression; - - public ExtractionResult(TupleDomain tupleDomain, RowExpression remainingExpression) - { - this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); - this.remainingExpression = requireNonNull(remainingExpression, "remainingExpression is null"); - } - - public TupleDomain getTupleDomain() - { - return tupleDomain; - } - - public RowExpression getRemainingExpression() - { - return remainingExpression; - } - } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index 1b8e6b24bd866..b89dd1ab0c85d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.predicate.Range; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.predicate.ValueSet; +import com.facebook.presto.spi.relation.DomainTranslator.ExtractionResult; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -28,7 +29,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.sql.relational.RowExpressionDomainTranslator.ExtractionResult; import com.facebook.presto.sql.relational.StandardFunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java new file mode 100644 index 0000000000000..c24e1ab381418 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.predicate.TupleDomain; + +import static java.util.Objects.requireNonNull; + +public interface DomainTranslator +{ + RowExpression toPredicate(TupleDomain tupleDomain); + + /** + * Convert a RowExpression predicate into an ExtractionResult consisting of: + * 1) A successfully extracted TupleDomain + * 2) An RowExpression fragment which represents the part of the original RowExpression that will need to be re-evaluated + * after filtering with the TupleDomain. + */ + ExtractionResult fromPredicate(ConnectorSession session, RowExpression predicate); + + class ExtractionResult + { + private final TupleDomain tupleDomain; + private final RowExpression remainingExpression; + + public ExtractionResult(TupleDomain tupleDomain, RowExpression remainingExpression) + { + this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); + this.remainingExpression = requireNonNull(remainingExpression, "remainingExpression is null"); + } + + public TupleDomain getTupleDomain() + { + return tupleDomain; + } + + public RowExpression getRemainingExpression() + { + return remainingExpression; + } + } +} From c2d7fdb7949c294a2721c7075d8c67c31f1a1d5e Mon Sep 17 00:00:00 2001 From: James Sun Date: Mon, 15 Apr 2019 11:47:33 -0700 Subject: [PATCH 33/66] Add DomainTranslator to ConnectorContext --- .../connector/ConnectorContextInstance.java | 12 ++++++- .../presto/connector/ConnectorManager.java | 10 ++++-- .../presto/server/ServerMainModule.java | 5 +++ .../ConnectorRowExpressionService.java | 36 +++++++++++++++++++ .../RowExpressionDomainTranslator.java | 2 ++ .../presto/testing/LocalQueryRunner.java | 4 ++- .../testing/TestingConnectorContext.java | 19 ++++++++++ .../spi/connector/ConnectorContext.java | 6 ++++ .../spi/relation/RowExpressionService.java | 22 ++++++++++++ 9 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java index ef3c3c0474224..c0467f33c5bc1 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import static java.util.Objects.requireNonNull; @@ -28,17 +29,20 @@ public class ConnectorContextInstance private final TypeManager typeManager; private final PageSorter pageSorter; private final PageIndexerFactory pageIndexerFactory; + private final RowExpressionService rowExpressionService; public ConnectorContextInstance( NodeManager nodeManager, TypeManager typeManager, PageSorter pageSorter, - PageIndexerFactory pageIndexerFactory) + PageIndexerFactory pageIndexerFactory, + RowExpressionService rowExpressionService) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); + this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); } @Override @@ -64,4 +68,10 @@ public PageIndexerFactory getPageIndexerFactory() { return pageIndexerFactory; } + + @Override + public RowExpressionService getRowExpressionService() + { + return rowExpressionService; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 4f4e9a6435a40..aa32629722b74 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -41,6 +41,7 @@ import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.procedure.Procedure; +import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.split.PageSinkManager; @@ -48,6 +49,7 @@ import com.facebook.presto.split.RecordPageSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.relational.ConnectorRowExpressionService; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -95,6 +97,7 @@ public class ConnectorManager private final PageIndexerFactory pageIndexerFactory; private final NodeInfo nodeInfo; private final TransactionManager transactionManager; + private final DomainTranslator domainTranslator; @GuardedBy("this") private final ConcurrentMap connectorFactories = new ConcurrentHashMap<>(); @@ -120,7 +123,8 @@ public ConnectorManager( TypeManager typeManager, PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, - TransactionManager transactionManager) + TransactionManager transactionManager, + DomainTranslator domainTranslator) { this.metadataManager = metadataManager; this.catalogManager = catalogManager; @@ -137,6 +141,7 @@ public ConnectorManager( this.pageIndexerFactory = pageIndexerFactory; this.nodeInfo = nodeInfo; this.transactionManager = transactionManager; + this.domainTranslator = domainTranslator; } @PreDestroy @@ -317,7 +322,8 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact new ConnectorAwareNodeManager(nodeManager, nodeInfo.getEnvironment(), connectorId), typeManager, pageSorter, - pageIndexerFactory); + pageIndexerFactory, + new ConnectorRowExpressionService(domainTranslator)); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(factory.getClass().getClassLoader())) { return factory.create(connectorId.getCatalogName(), properties, context); diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index a93f2f307db9a..b6b380df8e580 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -83,6 +83,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spiller.FileSingleStreamSpillerFactory; @@ -114,6 +115,7 @@ import com.facebook.presto.sql.planner.CompilerConfig; import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.transaction.TransactionManagerConfig; @@ -358,6 +360,9 @@ protected void setup(Binder binder) binder.bind(MetadataManager.class).in(Scopes.SINGLETON); binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON); + // row expression utils + binder.bind(DomainTranslator.class).to(RowExpressionDomainTranslator.class).in(Scopes.SINGLETON); + // type binder.bind(TypeRegistry.class).in(Scopes.SINGLETON); binder.bind(TypeManager.class).to(TypeRegistry.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java new file mode 100644 index 0000000000000..d8cb34c2b07af --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.RowExpressionService; + +import static java.util.Objects.requireNonNull; + +public final class ConnectorRowExpressionService + implements RowExpressionService +{ + private final DomainTranslator domainTranslator; + + public ConnectorRowExpressionService(DomainTranslator domainTranslator) + { + this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); + } + + @Override + public DomainTranslator getDomainTranslator() + { + return domainTranslator; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index 8a9955f3474f5..bb57e568d9348 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -49,6 +49,7 @@ import com.google.common.collect.PeekingIterator; import javax.annotation.Nullable; +import javax.inject.Inject; import java.util.ArrayList; import java.util.List; @@ -95,6 +96,7 @@ public final class RowExpressionDomainTranslator private final StandardFunctionResolution functionResolution; private final Metadata metadata; + @Inject public RowExpressionDomainTranslator(Metadata metadata) { this.metadata = requireNonNull(metadata, "metadata is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 53fd41b5775bc..956b53e02a44b 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -139,6 +139,7 @@ import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.planner.planPrinter.PlanPrinter; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateView; @@ -342,7 +343,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, typeRegistry, pageSorter, pageIndexerFactory, - transactionManager); + transactionManager, + new RowExpressionDomainTranslator(metadata)); GlobalSystemConnectorFactory globalSystemConnectorFactory = new GlobalSystemConnectorFactory(ImmutableSet.of( new NodeSystemTable(nodeManager), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 916c94e950244..6c9ea07901ed9 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -20,15 +20,19 @@ import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.InMemoryNodeManager; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.relation.DomainTranslator; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.type.TypeRegistry; public class TestingConnectorContext @@ -38,6 +42,8 @@ public class TestingConnectorContext private final TypeManager typeManager = new TypeRegistry(); private final PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); private final PageIndexerFactory pageIndexerFactory = new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())); + private final Metadata metadata = MetadataManager.createTestMetadataManager(); + private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); public TestingConnectorContext() { @@ -68,4 +74,17 @@ public PageIndexerFactory getPageIndexerFactory() { return pageIndexerFactory; } + + @Override + public RowExpressionService getRowExpressionService() + { + return new RowExpressionService() + { + @Override + public DomainTranslator getDomainTranslator() + { + return domainTranslator; + } + }; + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java index 886a9d88978fc..55d3305610174 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; public interface ConnectorContext @@ -39,4 +40,9 @@ default PageIndexerFactory getPageIndexerFactory() { throw new UnsupportedOperationException(); } + + default RowExpressionService getRowExpressionService() + { + throw new UnsupportedOperationException(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java new file mode 100644 index 0000000000000..d39a2a944d969 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +/** + * A set of services/utilities that are helpful for connectors to operate on row expressions + */ +public interface RowExpressionService +{ + DomainTranslator getDomainTranslator(); +} From afa296c4ce08d0212ad243773a7e0c171f73bd31 Mon Sep 17 00:00:00 2001 From: Wenlei Xie Date: Tue, 16 Apr 2019 16:50:48 -0700 Subject: [PATCH 34/66] Fix flaky test TestExchangeClient#testRemoveRemoteSource --- .../facebook/presto/operator/TestExchangeClient.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java index 2fb757448ec21..550d55dba16cb 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java @@ -365,8 +365,15 @@ public void testRemoveRemoteSource() // remove remote source while pages are still available exchangeClient.removeRemoteSource(taskId1); - // client should not receive any further pages from removed remote source - assertNull(exchangeClient.pollPage()); + SerializedPage nextPage = exchangeClient.pollPage(); + if (nextPage != null) { + // After the first page get polled, it is possible for the second page get buffered + // before remote source get removed. + assertPageEquals(nextPage, createPage(2)); + // Buffer capacity is set to 1 byte, so the third page cannot be buffered. + // Client should not receive any further pages from removed remote source. + assertNull(exchangeClient.pollPage()); + } assertEquals(exchangeClient.getStatus().getBufferedPages(), 0); // add pages to another source From 89d5f1a98547463aa5d66fe51941a493528b00fb Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 12:58:37 -0700 Subject: [PATCH 35/66] Move FunctionMetadata to SPI --- .../com/facebook/presto/cost/FilterStatsCalculator.java | 2 +- .../com/facebook/presto/cost/ScalarStatsCalculator.java | 2 +- .../com/facebook/presto/metadata/FunctionManager.java | 1 + .../com/facebook/presto/metadata/FunctionNamespace.java | 1 + .../presto/metadata/StaticFunctionNamespace.java | 1 + .../presto/operator/scalar/ArrayToArrayCast.java | 2 +- .../facebook/presto/sql/analyzer/ExpressionAnalyzer.java | 2 +- .../presto/sql/analyzer/WindowFunctionValidator.java | 2 +- .../com/facebook/presto/sql/gen/NullIfCodeGenerator.java | 2 +- .../presto/sql/planner/ExpressionInterpreter.java | 2 +- .../presto/sql/planner/RowExpressionInterpreter.java | 2 +- .../sql/planner/optimizations/WindowFilterPushDown.java | 2 +- .../sql/relational/RowExpressionDomainTranslator.java | 2 +- .../sql/relational/optimizer/ExpressionOptimizer.java | 2 +- .../sql/planner/assertions/RowExpressionVerifier.java | 2 +- .../facebook/presto/spi/function}/FunctionMetadata.java | 9 ++++----- 16 files changed, 19 insertions(+), 17 deletions(-) rename {presto-main/src/main/java/com/facebook/presto/metadata => presto-spi/src/main/java/com/facebook/presto/spi/function}/FunctionMetadata.java (91%) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index 091ca8687f2ba..8bb43de28852c 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 44d0f17f5edd5..7909f0ab634dc 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -15,8 +15,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index c7fd8985d5b3b..408f837d70567 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -19,6 +19,7 @@ import com.facebook.presto.operator.window.WindowFunctionSupplier; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java index 1921583429f57..20bf0729d1909 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; import com.facebook.presto.sql.tree.QualifiedName; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java index 4d9933a5d67c2..582bc81f65ac8 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java @@ -148,6 +148,7 @@ import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java index a4031aef877ed..9d046bb2be5b5 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java @@ -16,11 +16,11 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.CastType; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.SqlOperator; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.ArrayGeneratorUtils; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index a290bcfdc89cd..231cc4e75107f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -17,7 +17,6 @@ import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.OperatorNotFoundException; import com.facebook.presto.metadata.QualifiedObjectName; @@ -26,6 +25,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalParseResult; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java index 4963192de49ee..f5597d8d30e8f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/WindowFunctionValidator.java @@ -14,7 +14,7 @@ package com.facebook.presto.sql.analyzer; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.sql.tree.DefaultExpressionTraversalVisitor; import com.facebook.presto.sql.tree.FunctionCall; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java index 975e610bee1ff..f63039da41ea5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java @@ -15,9 +15,9 @@ import com.facebook.presto.metadata.CastType; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeSignature; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java index e066ae15dcdaa..c789c31ae4eeb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionInterpreter.java @@ -16,7 +16,6 @@ import com.facebook.presto.Session; import com.facebook.presto.client.FailureInfo; import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.operator.scalar.ArraySubscriptOperator; import com.facebook.presto.spi.ConnectorSession; @@ -24,6 +23,7 @@ import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RowBlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.FunctionType; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index e42fb9c789c33..393d498879dd7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -14,12 +14,12 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.client.FailureInfo; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RowBlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java index 6238f9d25b84d..d416da954c347 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Range; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index bb57e568d9348..e749f1bfa1577 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -14,12 +14,12 @@ package com.facebook.presto.sql.relational; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.OperatorNotFoundException; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.predicate.DiscreteValues; import com.facebook.presto.spi.predicate.Domain; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java index 25ba9d550481b..7310d8ac8747e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java @@ -15,10 +15,10 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index a7382bf40fc57..d5e010db0ba9d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.assertions; import com.facebook.presto.Session; -import com.facebook.presto.metadata.FunctionMetadata; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java similarity index 91% rename from presto-main/src/main/java/com/facebook/presto/metadata/FunctionMetadata.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java index b87642b8d3287..b4280fbd37c91 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java @@ -11,16 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; +package com.facebook.presto.spi.function; -import com.facebook.presto.spi.function.FunctionKind; -import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.TypeSignature; -import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static java.util.Collections.unmodifiableList; import static java.util.Objects.requireNonNull; public class FunctionMetadata @@ -66,7 +65,7 @@ private FunctionMetadata( { this.name = requireNonNull(name, "name is null"); this.operatorType = requireNonNull(operatorType, "operatorType is null"); - this.argumentTypes = ImmutableList.copyOf(requireNonNull(argumentTypes, "argumentTypes is null")); + this.argumentTypes = unmodifiableList(new ArrayList<>(requireNonNull(argumentTypes, "argumentTypes is null"))); this.returnType = requireNonNull(returnType, "returnType is null"); this.functionKind = requireNonNull(functionKind, "functionKind is null"); this.deterministic = deterministic; From 17d32381f565de26fe19ab929dc0b6e6a1e3b184 Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 13:00:22 -0700 Subject: [PATCH 36/66] Introduce FunctionMetadataManager --- .../presto/metadata/FunctionManager.java | 3 +++ .../spi/function/FunctionMetadataManager.java | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index 408f837d70567..d7018ce826564 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; @@ -34,6 +35,7 @@ @ThreadSafe public class FunctionManager + implements FunctionMetadataManager { private final StaticFunctionNamespace staticFunctionNamespace; private final FunctionInvokerProvider functionInvokerProvider; @@ -77,6 +79,7 @@ public FunctionHandle resolveFunction(Session session, QualifiedName name, List< return staticFunctionNamespace.resolveFunction(name, parameterTypes); } + @Override public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) { return staticFunctionNamespace.getFunctionMetadata(functionHandle); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java new file mode 100644 index 0000000000000..04231b2c15557 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +public interface FunctionMetadataManager +{ + FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); +} From 8907233cbc7a46aa2227919333e7949cb96db80c Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 13:05:11 -0700 Subject: [PATCH 37/66] Add FunctionMetadataManager to ConnectorContext --- .../presto/connector/ConnectorContextInstance.java | 10 ++++++++++ .../presto/connector/ConnectorManager.java | 1 + .../presto/testing/TestingConnectorContext.java | 14 ++++++++------ .../presto/spi/connector/ConnectorContext.java | 6 ++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java index c0467f33c5bc1..3e5a6fbb1d496 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -27,6 +28,7 @@ public class ConnectorContextInstance { private final NodeManager nodeManager; private final TypeManager typeManager; + private final FunctionMetadataManager functionMetadataManager; private final PageSorter pageSorter; private final PageIndexerFactory pageIndexerFactory; private final RowExpressionService rowExpressionService; @@ -34,12 +36,14 @@ public class ConnectorContextInstance public ConnectorContextInstance( NodeManager nodeManager, TypeManager typeManager, + FunctionMetadataManager functionMetadataManager, PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, RowExpressionService rowExpressionService) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); @@ -57,6 +61,12 @@ public TypeManager getTypeManager() return typeManager; } + @Override + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionMetadataManager; + } + @Override public PageSorter getPageSorter() { diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index aa32629722b74..e6774d18f972b 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -321,6 +321,7 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact ConnectorContext context = new ConnectorContextInstance( new ConnectorAwareNodeManager(nodeManager, nodeInfo.getEnvironment(), connectorId), typeManager, + metadataManager.getFunctionManager(), pageSorter, pageIndexerFactory, new ConnectorRowExpressionService(domainTranslator)); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 6c9ea07901ed9..189c7896903a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -40,17 +41,12 @@ public class TestingConnectorContext { private final NodeManager nodeManager = new ConnectorAwareNodeManager(new InMemoryNodeManager(), "testenv", new ConnectorId("test")); private final TypeManager typeManager = new TypeRegistry(); + private final FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); private final PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); private final PageIndexerFactory pageIndexerFactory = new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())); private final Metadata metadata = MetadataManager.createTestMetadataManager(); private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); - public TestingConnectorContext() - { - // associate typeManager with a function manager - new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - } - @Override public NodeManager getNodeManager() { @@ -63,6 +59,12 @@ public TypeManager getTypeManager() return typeManager; } + @Override + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionManager; + } + @Override public PageSorter getPageSorter() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java index 55d3305610174..d1fb3ea8e53d6 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -31,6 +32,11 @@ default TypeManager getTypeManager() throw new UnsupportedOperationException(); } + default FunctionMetadataManager getFunctionMetadataManager() + { + throw new UnsupportedOperationException(); + } + default PageSorter getPageSorter() { throw new UnsupportedOperationException(); From 493e1dec5946d17d6126209e721dff7cc8494435 Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 13:14:06 -0700 Subject: [PATCH 38/66] Rename StandardFunctionResolution to FunctionResolution --- .../presto/cost/FilterStatsCalculator.java | 6 ++--- .../presto/cost/ScalarStatsCalculator.java | 4 ++-- .../SimpleFilterProjectSemiJoinStatsRule.java | 6 ++--- .../sql/planner/RowExpressionInterpreter.java | 8 +++---- ...esolution.java => FunctionResolution.java} | 4 ++-- .../RowExpressionDomainTranslator.java | 10 ++++----- .../SqlToRowExpressionTranslator.java | 22 +++++++++---------- .../presto/cost/TestValuesNodeStats.java | 6 ++--- .../metadata/TestStaticFunctionNamespace.java | 4 ++-- .../TestRowExpressionDomainTranslator.java | 4 ++-- .../assertions/RowExpressionVerifier.java | 6 ++--- 11 files changed, 40 insertions(+), 40 deletions(-) rename presto-main/src/main/java/com/facebook/presto/sql/relational/{StandardFunctionResolution.java => FunctionResolution.java} (98%) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index 8bb43de28852c..36d91ee6cdca2 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -37,7 +37,7 @@ import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.AstVisitor; import com.facebook.presto.sql.tree.BetweenPredicate; import com.facebook.presto.sql.tree.BooleanLiteral; @@ -100,7 +100,7 @@ public class FilterStatsCalculator private final ScalarStatsCalculator scalarStatsCalculator; private final StatsNormalizer normalizer; private final LiteralEncoder literalEncoder; - private final StandardFunctionResolution functionResolution; + private final FunctionResolution functionResolution; public FilterStatsCalculator(Metadata metadata, ScalarStatsCalculator scalarStatsCalculator, StatsNormalizer normalizer) { @@ -108,7 +108,7 @@ public FilterStatsCalculator(Metadata metadata, ScalarStatsCalculator scalarStat this.scalarStatsCalculator = requireNonNull(scalarStatsCalculator, "scalarStatsCalculator is null"); this.normalizer = requireNonNull(normalizer, "normalizer is null"); this.literalEncoder = new LiteralEncoder(metadata.getBlockEncodingSerde()); - this.functionResolution = new StandardFunctionResolution(metadata.getFunctionManager()); + this.functionResolution = new FunctionResolution(metadata.getFunctionManager()); } @Deprecated diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 7909f0ab634dc..07ef1fe62b322 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -35,7 +35,7 @@ import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.AstVisitor; @@ -98,7 +98,7 @@ private class RowExpressionStatsVisitor { private final PlanNodeStatsEstimate input; private final Session session; - private final StandardFunctionResolution resolution = new StandardFunctionResolution(metadata.getFunctionManager()); + private final FunctionResolution resolution = new FunctionResolution(metadata.getFunctionManager()); public RowExpressionStatsVisitor(PlanNodeStatsEstimate input, Session session) { diff --git a/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java index 76b16e98bbdc1..afc49a44c435c 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java @@ -26,8 +26,8 @@ import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.LogicalRowExpressions; -import com.facebook.presto.sql.relational.StandardFunctionResolution; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NotExpression; import com.facebook.presto.sql.tree.SymbolReference; @@ -61,7 +61,7 @@ public class SimpleFilterProjectSemiJoinStatsRule private final FilterStatsCalculator filterStatsCalculator; private final LogicalRowExpressions logicalRowExpressions; - private final StandardFunctionResolution functionResolution; + private final FunctionResolution functionResolution; public SimpleFilterProjectSemiJoinStatsRule(StatsNormalizer normalizer, FilterStatsCalculator filterStatsCalculator, FunctionManager functionManager) { @@ -69,7 +69,7 @@ public SimpleFilterProjectSemiJoinStatsRule(StatsNormalizer normalizer, FilterSt this.filterStatsCalculator = requireNonNull(filterStatsCalculator, "filterStatsCalculator can not be null"); requireNonNull(functionManager, "functionManager can not be null"); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); - this.functionResolution = new StandardFunctionResolution(functionManager); + this.functionResolution = new FunctionResolution(functionManager); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 393d498879dd7..58632aa214e93 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -36,7 +36,7 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.InterpretedFunctionInvoker; import com.facebook.presto.sql.planner.Interpreters.LambdaSymbolResolver; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.util.Failures; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -106,7 +106,7 @@ public class RowExpressionInterpreter private final boolean optimize; private final InterpretedFunctionInvoker functionInvoker; private final com.facebook.presto.sql.relational.DeterminismEvaluator determinismEvaluator; - private final StandardFunctionResolution resolution; + private final FunctionResolution resolution; private final Visitor visitor; @@ -131,7 +131,7 @@ public RowExpressionInterpreter(RowExpression expression, Metadata metadata, Con this.optimize = optimize; this.functionInvoker = new InterpretedFunctionInvoker(metadata.getFunctionManager()); this.determinismEvaluator = new com.facebook.presto.sql.relational.DeterminismEvaluator(metadata.getFunctionManager()); - this.resolution = new StandardFunctionResolution(metadata.getFunctionManager()); + this.resolution = new FunctionResolution(metadata.getFunctionManager()); this.visitor = new Visitor(); } @@ -719,7 +719,7 @@ private SpecialCallResult tryHandleCast(CallExpression callExpression, List argumentValues, List argumentTypes, Object context) { - StandardFunctionResolution resolution = new StandardFunctionResolution(metadata.getFunctionManager()); + FunctionResolution resolution = new FunctionResolution(metadata.getFunctionManager()); checkArgument(resolution.isLikeFunction(callExpression.getFunctionHandle())); checkArgument(callExpression.getArguments().size() == 2); RowExpression likePatternExpression = callExpression.getArguments().get(1); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java similarity index 98% rename from presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java rename to presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index 554dacff5ac8f..7e82eeee9c9ce 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/StandardFunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -47,11 +47,11 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; -public final class StandardFunctionResolution +public final class FunctionResolution { private final FunctionManager functionManager; - public StandardFunctionResolution(FunctionManager functionManager) + public FunctionResolution(FunctionManager functionManager) { this.functionManager = requireNonNull(functionManager, "functionManager is null"); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index e749f1bfa1577..69de2173e5981 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -93,8 +93,8 @@ public final class RowExpressionDomainTranslator { private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; - private final StandardFunctionResolution functionResolution; private final Metadata metadata; + private final FunctionResolution functionResolution; @Inject public RowExpressionDomainTranslator(Metadata metadata) @@ -102,7 +102,7 @@ public RowExpressionDomainTranslator(Metadata metadata) this.metadata = requireNonNull(metadata, "metadata is null"); this.functionManager = metadata.getFunctionManager(); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); - this.functionResolution = new StandardFunctionResolution(functionManager); + this.functionResolution = new FunctionResolution(functionManager); } @Override @@ -298,7 +298,7 @@ private static class Visitor private final FunctionManager functionManager; private final LogicalRowExpressions logicalRowExpressions; private final DeterminismEvaluator determinismEvaluator; - private final StandardFunctionResolution resolution; + private final FunctionResolution resolution; private Visitor(Metadata metadata, ConnectorSession session) { @@ -308,7 +308,7 @@ private Visitor(Metadata metadata, ConnectorSession session) this.functionManager = metadata.getFunctionManager(); this.logicalRowExpressions = new LogicalRowExpressions(functionManager); this.determinismEvaluator = new DeterminismEvaluator(functionManager); - this.resolution = new StandardFunctionResolution(functionManager); + this.resolution = new FunctionResolution(functionManager); } @Override @@ -809,7 +809,7 @@ private static RowExpression isNull(RowExpression expression) return new SpecialFormExpression(IS_NULL, BOOLEAN, expression); } - private static RowExpression not(StandardFunctionResolution resolution, RowExpression expression) + private static RowExpression not(FunctionResolution resolution, RowExpression expression) { return call("not", resolution.notFunction(), expression.getType(), expression); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java index 4344fe3d12182..c10cb00110969 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java @@ -181,7 +181,7 @@ private static class Visitor private final TypeManager typeManager; private final FunctionManager functionManager; private final Session session; - private final StandardFunctionResolution standardFunctionResolution; + private final FunctionResolution functionResolution; private Visitor( Map, Type> types, @@ -195,7 +195,7 @@ private Visitor( this.typeManager = typeManager; this.functionManager = functionManager; this.session = session; - this.standardFunctionResolution = new StandardFunctionResolution(functionManager); + this.functionResolution = new FunctionResolution(functionManager); } private Type getType(Expression node) @@ -365,7 +365,7 @@ protected RowExpression visitComparisonExpression(ComparisonExpression node, Voi return call( node.getOperator().name(), - standardFunctionResolution.comparisonFunction(node.getOperator(), left.getType(), right.getType()), + functionResolution.comparisonFunction(node.getOperator(), left.getType(), right.getType()), BOOLEAN, left, right); @@ -438,7 +438,7 @@ protected RowExpression visitArithmeticBinary(ArithmeticBinaryExpression node, V return call( node.getOperator().name(), - standardFunctionResolution.arithmeticFunction(node.getOperator(), left.getType(), right.getType()), + functionResolution.arithmeticFunction(node.getOperator(), left.getType(), right.getType()), getType(node), left, right); @@ -670,7 +670,7 @@ protected RowExpression visitIfExpression(IfExpression node, Void context) @Override protected RowExpression visitTryExpression(TryExpression node, Void context) { - return call("TRY", standardFunctionResolution.tryFunction(getType(node)), getType(node), process(node.getInnerExpression(), context)); + return call("TRY", functionResolution.tryFunction(getType(node)), getType(node), process(node.getInnerExpression(), context)); } @Override @@ -693,7 +693,7 @@ protected RowExpression visitIsNotNullPredicate(IsNotNullPredicate node, Void co return call( "not", - standardFunctionResolution.notFunction(), + functionResolution.notFunction(), BOOLEAN, specialForm(IS_NULL, BOOLEAN, ImmutableList.of(expression))); } @@ -709,7 +709,7 @@ protected RowExpression visitIsNullPredicate(IsNullPredicate node, Void context) @Override protected RowExpression visitNotExpression(NotExpression node, Void context) { - return call("not", standardFunctionResolution.notFunction(), BOOLEAN, process(node.getValue(), context)); + return call("not", functionResolution.notFunction(), BOOLEAN, process(node.getValue(), context)); } @Override @@ -745,7 +745,7 @@ protected RowExpression visitLikePredicate(LikePredicate node, Void context) if (node.getEscape().isPresent()) { RowExpression escape = process(node.getEscape().get(), context); - return likeFunctionCall(value, call("LIKE_PATTERN", standardFunctionResolution.likePatternFunction(), LIKE_PATTERN, pattern, escape)); + return likeFunctionCall(value, call("LIKE_PATTERN", functionResolution.likePatternFunction(), LIKE_PATTERN, pattern, escape)); } return likeFunctionCall(value, call(CAST.name(), functionManager.lookupCast(CAST, VARCHAR.getTypeSignature(), LIKE_PATTERN.getTypeSignature()), LIKE_PATTERN, pattern)); @@ -754,11 +754,11 @@ protected RowExpression visitLikePredicate(LikePredicate node, Void context) private RowExpression likeFunctionCall(RowExpression value, RowExpression pattern) { if (value.getType() instanceof VarcharType) { - return call("LIKE", standardFunctionResolution.likeVarcharFunction(), BOOLEAN, value, pattern); + return call("LIKE", functionResolution.likeVarcharFunction(), BOOLEAN, value, pattern); } checkState(value.getType() instanceof CharType, "LIKE value type is neither VARCHAR or CHAR"); - return call("LIKE", standardFunctionResolution.likeCharFunction(value.getType()), BOOLEAN, value, pattern); + return call("LIKE", functionResolution.likeCharFunction(value.getType()), BOOLEAN, value, pattern); } @Override @@ -784,7 +784,7 @@ protected RowExpression visitArrayConstructor(ArrayConstructor node, Void contex List argumentTypes = arguments.stream() .map(RowExpression::getType) .collect(toImmutableList()); - return call("ARRAY", standardFunctionResolution.arrayConstructor(argumentTypes), getType(node), arguments); + return call("ARRAY", functionResolution.arrayConstructor(argumentTypes), getType(node), arguments); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java b/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java index 306afbc990661..e0c8799f8f230 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java @@ -14,7 +14,7 @@ package com.facebook.presto.cost; import com.facebook.presto.sql.planner.Symbol; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -35,7 +35,7 @@ public class TestValuesNodeStats @Test public void testStatsForValuesNode() { - StandardFunctionResolution resolution = new StandardFunctionResolution(tester().getMetadata().getFunctionManager()); + FunctionResolution resolution = new FunctionResolution(tester().getMetadata().getFunctionManager()); tester().assertStatsFor(pb -> pb .values(ImmutableList.of(pb.symbol("a", BIGINT), pb.symbol("b", DOUBLE)), ImmutableList.of( @@ -86,7 +86,7 @@ public void testStatsForValuesNode() @Test public void testStatsForValuesNodeWithJustNulls() { - StandardFunctionResolution resolution = new StandardFunctionResolution(tester().getMetadata().getFunctionManager()); + FunctionResolution resolution = new FunctionResolution(tester().getMetadata().getFunctionManager()); PlanNodeStatsEstimate nullAStats = PlanNodeStatsEstimate.builder() .setOutputRowCount(1) .addSymbolStatistics(new Symbol("a"), SymbolStatsEstimate.zero()) diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java index c71cdd96a9972..b016e5c134fe1 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java @@ -27,7 +27,7 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; @@ -162,7 +162,7 @@ public void testOperatorTypes() { TypeRegistry typeManager = new TypeRegistry(); FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - StandardFunctionResolution functionResolution = new StandardFunctionResolution(functionManager); + FunctionResolution functionResolution = new FunctionResolution(functionManager); assertTrue(functionManager.getFunctionMetadata(functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT)).getOperatorType().map(OperatorType::isArithmeticOperator).orElse(false)); assertFalse(functionManager.getFunctionMetadata(functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT)).getOperatorType().map(OperatorType::isComparisonOperator).orElse(true)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index b89dd1ab0c85d..aa6b7adf03c98 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -28,8 +28,8 @@ import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.sql.relational.StandardFunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; @@ -1225,7 +1225,7 @@ private RowExpression cast(RowExpression expression, Type toType) private RowExpression not(RowExpression expression) { - return call("not", new StandardFunctionResolution(metadata.getFunctionManager()).notFunction(), expression.getType(), expression); + return call("not", new FunctionResolution(metadata.getFunctionManager()).notFunction(), expression.getType(), expression); } private RowExpression in(RowExpression value, List inList) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index d5e010db0ba9d..c0d733ba538a2 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -23,7 +23,7 @@ import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.LiteralInterpreter; -import com.facebook.presto.sql.relational.StandardFunctionResolution; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.AstVisitor; import com.facebook.presto.sql.tree.BetweenPredicate; @@ -91,14 +91,14 @@ final class RowExpressionVerifier private final SymbolAliases symbolAliases; private final Metadata metadata; private final Session session; - private final StandardFunctionResolution functionResolution; + private final FunctionResolution functionResolution; RowExpressionVerifier(SymbolAliases symbolAliases, Metadata metadata, Session session) { this.symbolAliases = requireNonNull(symbolAliases, "symbolLayout is null"); this.metadata = requireNonNull(metadata, "metadata is null"); this.session = requireNonNull(session, "session is null"); - this.functionResolution = new StandardFunctionResolution(metadata.getFunctionManager()); + this.functionResolution = new FunctionResolution(metadata.getFunctionManager()); } @Override From 4626852ebfa95ecc20fe99119f5e46eebe06ca11 Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 20:35:41 -0700 Subject: [PATCH 39/66] Add StandardFunctionResolution as an interface --- .../sql/relational/FunctionResolution.java | 66 ++++++++++++++++- .../relational/TestFunctionResolution.java | 73 +++++++++++++++++++ .../function/StandardFunctionResolution.java | 51 +++++++++++++ 3 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index 7e82eeee9c9ce..1506940d0bc55 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -16,6 +16,7 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; @@ -38,6 +39,7 @@ import static com.facebook.presto.spi.function.OperatorType.MULTIPLY; import static com.facebook.presto.spi.function.OperatorType.NEGATION; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.function.OperatorType.SUBTRACT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; @@ -45,9 +47,11 @@ import static com.facebook.presto.sql.tree.ArrayConstructor.ARRAY_CONSTRUCTOR; import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public final class FunctionResolution + implements StandardFunctionResolution { private final FunctionManager functionManager; @@ -56,6 +60,7 @@ public FunctionResolution(FunctionManager functionManager) this.functionManager = requireNonNull(functionManager, "functionManager is null"); } + @Override public FunctionHandle notFunction() { return functionManager.lookupFunction("not", fromTypes(BOOLEAN)); @@ -66,11 +71,13 @@ public boolean isNotFunction(FunctionHandle functionHandle) return notFunction().equals(functionHandle); } + @Override public FunctionHandle likeVarcharFunction() { return functionManager.lookupFunction("LIKE", fromTypes(VARCHAR, LIKE_PATTERN)); } + @Override public FunctionHandle likeCharFunction(Type valueType) { checkArgument(valueType instanceof CharType, "Expected CHAR value type"); @@ -92,11 +99,25 @@ public boolean isCastFunction(FunctionHandle functionHandle) return functionManager.getFunctionMetadata(functionHandle).getName().equals(mangleOperatorName(OperatorType.CAST.name())); } + @Override + public FunctionHandle betweenFunction(Type valueType, Type lowerBoundType, Type upperBoundType) + { + return functionManager.lookupFunction(BETWEEN.getFunctionName(), fromTypes(valueType, lowerBoundType, upperBoundType)); + } + + @Override public boolean isBetweenFunction(FunctionHandle functionHandle) { return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(BETWEEN)); } + @Override + public FunctionHandle arithmeticFunction(OperatorType operator, Type leftType, Type rightType) + { + checkArgument(operator.isArithmeticOperator(), format("unexpected arithmetic type %s", operator)); + return functionManager.resolveOperator(operator, fromTypes(leftType, rightType)); + } + public FunctionHandle arithmeticFunction(ArithmeticBinaryExpression.Operator operator, Type leftType, Type rightType) { OperatorType operatorType; @@ -119,19 +140,41 @@ public FunctionHandle arithmeticFunction(ArithmeticBinaryExpression.Operator ope default: throw new IllegalStateException("Unknown arithmetic operator: " + operator); } - return functionManager.resolveOperator(operatorType, fromTypes(leftType, rightType)); + return arithmeticFunction(operatorType, leftType, rightType); + } + + @Override + public boolean isArithmeticFunction(FunctionHandle functionHandle) + { + Optional operatorType = functionManager.getFunctionMetadata(functionHandle).getOperatorType(); + return operatorType.isPresent() && operatorType.get().isArithmeticOperator(); + } + + @Override + public FunctionHandle negateFunction(Type type) + { + return functionManager.lookupFunction(NEGATION.getFunctionName(), fromTypes(type)); } + @Override public boolean isNegateFunction(FunctionHandle functionHandle) { return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(NEGATION)); } + @Override public FunctionHandle arrayConstructor(List argumentTypes) { return functionManager.lookupFunction(ARRAY_CONSTRUCTOR, fromTypes(argumentTypes)); } + @Override + public FunctionHandle comparisonFunction(OperatorType operator, Type leftType, Type rightType) + { + checkArgument(operator.isComparisonOperator(), format("unexpected comparison type %s", operator)); + return functionManager.resolveOperator(operator, fromTypes(leftType, rightType)); + } + public FunctionHandle comparisonFunction(ComparisonExpression.Operator operator, Type leftType, Type rightType) { OperatorType operatorType; @@ -161,7 +204,26 @@ public FunctionHandle comparisonFunction(ComparisonExpression.Operator operator, throw new IllegalStateException("Unsupported comparison operator type: " + operator); } - return functionManager.resolveOperator(operatorType, fromTypes(leftType, rightType)); + return comparisonFunction(operatorType, leftType, rightType); + } + + @Override + public boolean isComparisonFunction(FunctionHandle functionHandle) + { + Optional operatorType = functionManager.getFunctionMetadata(functionHandle).getOperatorType(); + return operatorType.isPresent() && operatorType.get().isComparisonOperator(); + } + + @Override + public FunctionHandle subscriptFunction(Type baseType, Type indexType) + { + return functionManager.lookupFunction(SUBSCRIPT.getFunctionName(), fromTypes(baseType, indexType)); + } + + @Override + public boolean isSubscriptFunction(FunctionHandle functionHandle) + { + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(SUBSCRIPT)); } public FunctionHandle tryFunction(Type returnType) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java new file mode 100644 index 0000000000000..4688e519d1a35 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestFunctionResolution.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.type.TypeRegistry; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestFunctionResolution +{ + private FunctionResolution functionResolution; + + @BeforeClass + public void setup() + { + TypeManager typeManager = new TypeRegistry(); + FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + functionResolution = new FunctionResolution(functionManager); + } + + @Test + public void testStandardFunctionResolution() + { + StandardFunctionResolution standardFunctionResolution = functionResolution; + + // not + assertTrue(standardFunctionResolution.isNotFunction(standardFunctionResolution.notFunction())); + + // negate + assertTrue(standardFunctionResolution.isNegateFunction(standardFunctionResolution.negateFunction(DOUBLE))); + assertFalse(standardFunctionResolution.isNotFunction(standardFunctionResolution.negateFunction(DOUBLE))); + + // arithmetic + assertTrue(standardFunctionResolution.isArithmeticFunction(standardFunctionResolution.arithmeticFunction(ADD, DOUBLE, DOUBLE))); + assertFalse(standardFunctionResolution.isComparisonFunction(standardFunctionResolution.arithmeticFunction(ADD, DOUBLE, DOUBLE))); + + // comparison + assertTrue(standardFunctionResolution.isComparisonFunction(standardFunctionResolution.comparisonFunction(GREATER_THAN, DOUBLE, DOUBLE))); + assertFalse(standardFunctionResolution.isArithmeticFunction(standardFunctionResolution.comparisonFunction(GREATER_THAN, DOUBLE, DOUBLE))); + + // between + assertTrue(standardFunctionResolution.isBetweenFunction(standardFunctionResolution.betweenFunction(DOUBLE, DOUBLE, DOUBLE))); + assertFalse(standardFunctionResolution.isNotFunction(standardFunctionResolution.betweenFunction(DOUBLE, DOUBLE, DOUBLE))); + + // subscript + assertTrue(standardFunctionResolution.isSubscriptFunction(standardFunctionResolution.subscriptFunction(new ArrayType(DOUBLE), BIGINT))); + assertFalse(standardFunctionResolution.isBetweenFunction(standardFunctionResolution.subscriptFunction(new ArrayType(DOUBLE), BIGINT))); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java new file mode 100644 index 0000000000000..6cb935d3bd13e --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.type.Type; + +import java.util.List; + +public interface StandardFunctionResolution +{ + FunctionHandle notFunction(); + + boolean isNotFunction(FunctionHandle functionHandle); + + FunctionHandle negateFunction(Type type); + + boolean isNegateFunction(FunctionHandle functionHandle); + + FunctionHandle likeVarcharFunction(); + + FunctionHandle likeCharFunction(Type valueType); + + FunctionHandle arrayConstructor(List argumentTypes); + + FunctionHandle arithmeticFunction(OperatorType operator, Type leftType, Type rightType); + + boolean isArithmeticFunction(FunctionHandle functionHandle); + + FunctionHandle comparisonFunction(OperatorType operator, Type leftType, Type rightType); + + boolean isComparisonFunction(FunctionHandle functionHandle); + + FunctionHandle betweenFunction(Type valueType, Type lowerBoundType, Type upperBoundType); + + boolean isBetweenFunction(FunctionHandle functionHandle); + + FunctionHandle subscriptFunction(Type baseType, Type indexType); + + boolean isSubscriptFunction(FunctionHandle functionHandle); +} From b926bde04d8ce761f42f5713b51322deb1752de0 Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 20:44:15 -0700 Subject: [PATCH 40/66] Add StandardFunctionResolution to ConnectorContext --- .../presto/connector/ConnectorContextInstance.java | 10 ++++++++++ .../facebook/presto/connector/ConnectorManager.java | 2 ++ .../presto/testing/TestingConnectorContext.java | 9 +++++++++ .../presto/spi/connector/ConnectorContext.java | 6 ++++++ 4 files changed, 27 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java index 3e5a6fbb1d496..3d13e18766c39 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -29,6 +30,7 @@ public class ConnectorContextInstance private final NodeManager nodeManager; private final TypeManager typeManager; private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution functionResolution; private final PageSorter pageSorter; private final PageIndexerFactory pageIndexerFactory; private final RowExpressionService rowExpressionService; @@ -37,6 +39,7 @@ public ConnectorContextInstance( NodeManager nodeManager, TypeManager typeManager, FunctionMetadataManager functionMetadataManager, + StandardFunctionResolution functionResolution, PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, RowExpressionService rowExpressionService) @@ -44,6 +47,7 @@ public ConnectorContextInstance( this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); @@ -67,6 +71,12 @@ public FunctionMetadataManager getFunctionMetadataManager() return functionMetadataManager; } + @Override + public StandardFunctionResolution getStandardFunctionResolution() + { + return functionResolution; + } + @Override public PageSorter getPageSorter() { diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index e6774d18f972b..ba256ec3049a8 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -50,6 +50,7 @@ import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.relational.ConnectorRowExpressionService; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -322,6 +323,7 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact new ConnectorAwareNodeManager(nodeManager, nodeInfo.getEnvironment(), connectorId), typeManager, metadataManager.getFunctionManager(), + new FunctionResolution(metadataManager.getFunctionManager()), pageSorter, pageIndexerFactory, new ConnectorRowExpressionService(domainTranslator)); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 189c7896903a1..2b9a90565dac3 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -28,11 +28,13 @@ import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.type.TypeRegistry; @@ -42,6 +44,7 @@ public class TestingConnectorContext private final NodeManager nodeManager = new ConnectorAwareNodeManager(new InMemoryNodeManager(), "testenv", new ConnectorId("test")); private final TypeManager typeManager = new TypeRegistry(); private final FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + private final StandardFunctionResolution functionResolution = new FunctionResolution(functionManager); private final PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); private final PageIndexerFactory pageIndexerFactory = new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())); private final Metadata metadata = MetadataManager.createTestMetadataManager(); @@ -65,6 +68,12 @@ public FunctionMetadataManager getFunctionMetadataManager() return functionManager; } + @Override + public StandardFunctionResolution getStandardFunctionResolution() + { + return functionResolution; + } + @Override public PageSorter getPageSorter() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java index d1fb3ea8e53d6..b966f6bd09260 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -37,6 +38,11 @@ default FunctionMetadataManager getFunctionMetadataManager() throw new UnsupportedOperationException(); } + default StandardFunctionResolution getStandardFunctionResolution() + { + throw new UnsupportedOperationException(); + } + default PageSorter getPageSorter() { throw new UnsupportedOperationException(); From e7e81911ccb6e5ed2b0de49d465417fd489e97ea Mon Sep 17 00:00:00 2001 From: Wenlei Xie Date: Wed, 17 Apr 2019 20:49:46 -0700 Subject: [PATCH 41/66] Move removeRemoteSource from RemoteTaskFactory to RemoteTask --- .../MemoryTrackingRemoteTaskFactory.java | 7 -- .../facebook/presto/execution/RemoteTask.java | 2 + .../presto/execution/RemoteTaskFactory.java | 3 - .../server/remotetask/HttpRemoteTask.java | 85 ++++++++++++++++++ .../remotetask/HttpRemoteTaskFactory.java | 86 ------------------- .../execution/MockRemoteTaskFactory.java | 12 +-- 6 files changed, 93 insertions(+), 102 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java index 0be7b5baee923..852dc75e0e215 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java @@ -22,7 +22,6 @@ import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.ListenableFuture; import java.util.OptionalInt; @@ -65,12 +64,6 @@ public RemoteTask createRemoteTask(Session session, return task; } - @Override - public ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId) - { - return remoteTaskFactory.removeRemoteSource(taskId, remoteSourceTaskId); - } - private static final class UpdatePeakMemory implements StateChangeListener { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java index 45e1f6841554d..07045acdaabfa 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTask.java @@ -40,6 +40,8 @@ public interface RemoteTask void setOutputBuffers(OutputBuffers outputBuffers); + ListenableFuture removeRemoteSource(TaskId remoteSourceTaskId); + /** * Listener is always notified asynchronously using a dedicated notification thread pool so, care should * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java index b6fbe0df63b93..e06c5d6088954 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java @@ -21,7 +21,6 @@ import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.ListenableFuture; import java.util.OptionalInt; @@ -36,6 +35,4 @@ RemoteTask createRemoteTask(Session session, OutputBuffers outputBuffers, PartitionedSplitCountTracker partitionedSplitCountTracker, boolean summarizeTaskInfo); - - ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java index d1aae612c7365..c496258a745ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java @@ -34,6 +34,7 @@ import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.PlanNodeId; @@ -45,15 +46,18 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import io.airlift.concurrent.SetThreadName; import io.airlift.http.client.HttpClient; import io.airlift.http.client.HttpUriBuilder; import io.airlift.http.client.Request; import io.airlift.http.client.ResponseHandler; +import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.log.Logger; import io.airlift.units.Duration; import org.joda.time.DateTime; +import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import java.net.URI; @@ -85,16 +89,21 @@ import static com.facebook.presto.server.smile.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler; import static com.facebook.presto.server.smile.FullSmileResponseHandler.createFullSmileResponseHandler; import static com.facebook.presto.server.smile.JsonCodecWrapper.unwrapJsonCodec; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.util.Failures.toFailure; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.util.concurrent.Futures.addCallback; import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static io.airlift.http.client.HttpStatus.OK; import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static io.airlift.http.client.Request.Builder.prepareDelete; import static io.airlift.http.client.Request.Builder.preparePost; import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -106,6 +115,7 @@ public final class HttpRemoteTask private static final Logger log = Logger.get(HttpRemoteTask.class); private final TaskId taskId; + private final URI taskLocation; private final Session session; private final String nodeId; @@ -114,6 +124,7 @@ public final class HttpRemoteTask private final AtomicLong nextSplitId = new AtomicLong(); + private final Duration maxErrorDuration; private final RemoteTaskStats stats; private final TaskInfoFetcher taskInfoFetcher; private final ContinuousTaskStatusFetcher taskStatusFetcher; @@ -197,10 +208,12 @@ public HttpRemoteTask(Session session, requireNonNull(taskInfoCodec, "taskInfoCodec is null"); requireNonNull(taskUpdateRequestCodec, "taskUpdateRequestCodec is null"); requireNonNull(partitionedSplitCountTracker, "partitionedSplitCountTracker is null"); + requireNonNull(maxErrorDuration, "maxErrorDuration is null"); requireNonNull(stats, "stats is null"); try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) { this.taskId = taskId; + this.taskLocation = location; this.session = session; this.nodeId = nodeId; this.planFragment = planFragment; @@ -214,6 +227,7 @@ public HttpRemoteTask(Session session, this.taskUpdateRequestCodec = taskUpdateRequestCodec; this.updateErrorTracker = new RequestErrorTracker(taskId, location, maxErrorDuration, errorScheduledExecutor, "updating task"); this.partitionedSplitCountTracker = requireNonNull(partitionedSplitCountTracker, "partitionedSplitCountTracker is null"); + this.maxErrorDuration = maxErrorDuration; this.stats = stats; this.isBinaryTransportEnabled = isBinaryTransportEnabled; @@ -382,6 +396,77 @@ public synchronized void setOutputBuffers(OutputBuffers newOutputBuffers) } } + @Override + public ListenableFuture removeRemoteSource(TaskId remoteSourceTaskId) + { + URI remoteSourceUri = uriBuilderFrom(taskLocation) + .appendPath("remote-source") + .appendPath(remoteSourceTaskId.toString()) + .build(); + + Request request = prepareDelete() + .setUri(remoteSourceUri) + .build(); + RequestErrorTracker errorTracker = new RequestErrorTracker( + taskId, + remoteSourceUri, + maxErrorDuration, + errorScheduledExecutor, + "Remove exchange remote source"); + + SettableFuture future = SettableFuture.create(); + doRemoveRemoteSource(errorTracker, request, future); + return future; + } + + /// This method may call itself recursively when retrying for failures + private void doRemoveRemoteSource(RequestErrorTracker errorTracker, Request request, SettableFuture future) + { + errorTracker.startRequest(); + + FutureCallback callback = new FutureCallback() { + @Override + public void onSuccess(@Nullable StatusResponse response) + { + if (response == null) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with null response"); + } + if (response.getStatusCode() != OK.code()) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with HTTP status " + response.getStatusCode()); + } + future.set(null); + } + + @Override + public void onFailure(Throwable failedReason) + { + if (failedReason instanceof RejectedExecutionException && httpClient.isClosed()) { + log.error("Unable to destroy exchange source at %s. HTTP client is closed", request.getUri()); + future.setException(failedReason); + return; + } + // record failure + try { + errorTracker.requestFailed(failedReason); + } + catch (PrestoException e) { + future.setException(e); + return; + } + // if throttled due to error, asynchronously wait for timeout and try again + ListenableFuture errorRateLimit = errorTracker.acquireRequestPermit(); + if (errorRateLimit.isDone()) { + doRemoveRemoteSource(errorTracker, request, future); + } + else { + errorRateLimit.addListener(() -> doRemoveRemoteSource(errorTracker, request, future), errorScheduledExecutor); + } + } + }; + + addCallback(httpClient.executeAsync(request, createStatusResponseHandler()), callback, directExecutor()); + } + @Override public int getPartitionedSplitCount() { diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java index 7c9936d8a3d38..4f334781243a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java @@ -31,45 +31,29 @@ import com.facebook.presto.server.TaskUpdateRequest; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; -import com.facebook.presto.spi.PrestoException; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.collect.Multimap; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; import io.airlift.concurrent.BoundedExecutor; import io.airlift.concurrent.ThreadPoolExecutorMBean; import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; -import javax.annotation.Nullable; import javax.annotation.PreDestroy; import javax.inject.Inject; -import java.net.URI; import java.util.OptionalInt; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import static com.facebook.presto.server.smile.JsonCodecWrapper.wrapJsonCodec; -import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static com.google.common.util.concurrent.Futures.addCallback; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; @@ -184,74 +168,4 @@ public RemoteTask createRemoteTask(Session session, stats, isBinaryTransportEnabled); } - - @Override - public ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId) - { - URI remoteSourceUri = uriBuilderFrom(locationFactory.createLocalTaskLocation(taskId)) - .appendPath("remote-source") - .appendPath(remoteSourceTaskId.toString()) - .build(); - - Request request = prepareDelete() - .setUri(remoteSourceUri) - .build(); - RequestErrorTracker errorTracker = new RequestErrorTracker( - taskId, - remoteSourceUri, - maxErrorDuration, - errorScheduledExecutor, - "Remove exchange remote source"); - - SettableFuture future = SettableFuture.create(); - removeRemoteSourceHelper(errorTracker, request, future); - return future; - } - - private void removeRemoteSourceHelper(RequestErrorTracker errorTracker, Request request, SettableFuture future) - { - errorTracker.startRequest(); - - FutureCallback callback = new FutureCallback() { - @Override - public void onSuccess(@Nullable StatusResponse response) - { - if (response == null) { - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with null response"); - } - if (response.getStatusCode() != OK.code()) { - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Request failed with HTTP status " + response.getStatusCode()); - } - future.set(null); - } - - @Override - public void onFailure(Throwable failedReason) - { - if (failedReason instanceof RejectedExecutionException && httpClient.isClosed()) { - LOG.error("Unable to destroy exchange source at %s. HTTP client is closed", request.getUri()); - future.setException(failedReason); - return; - } - // record failure - try { - errorTracker.requestFailed(failedReason); - } - catch (PrestoException e) { - future.setException(e); - return; - } - // if throttled due to error, asynchronously wait for timeout and try again - ListenableFuture errorRateLimit = errorTracker.acquireRequestPermit(); - if (errorRateLimit.isDone()) { - removeRemoteSourceHelper(errorTracker, request, future); - } - else { - errorRateLimit.addListener(() -> removeRemoteSourceHelper(errorTracker, request, future), errorScheduledExecutor); - } - } - }; - - addCallback(httpClient.executeAsync(request, createStatusResponseHandler()), callback, directExecutor()); - } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index e5cbe71b92451..e2bbd622ae789 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -143,12 +143,6 @@ public MockRemoteTask createRemoteTask( return new MockRemoteTask(taskId, fragment, node.getNodeIdentifier(), executor, scheduledExecutor, initialSplits, totalPartitions, partitionedSplitCountTracker); } - @Override - public ListenableFuture removeRemoteSource(TaskId taskId, TaskId remoteSourceTaskId) - { - throw new UnsupportedOperationException(); - } - public static final class MockRemoteTask implements RemoteTask { @@ -374,6 +368,12 @@ public void setOutputBuffers(OutputBuffers outputBuffers) outputBuffer.setOutputBuffers(outputBuffers); } + @Override + public ListenableFuture removeRemoteSource(TaskId remoteSourceTaskId) + { + throw new UnsupportedOperationException(); + } + @Override public void addStateChangeListener(StateChangeListener stateChangeListener) { From a01845abec3f4182ab490ba7941d9e9af4cffa8d Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 12:21:07 -0700 Subject: [PATCH 42/66] Apply expression optimization for RowExpressionInterpreter::optimize --- .../presto/sql/planner/RowExpressionInterpreter.java | 11 +++++++++-- .../sql/relational/SqlToRowExpressionTranslator.java | 2 +- .../sql/relational/optimizer/ExpressionOptimizer.java | 5 ++--- .../presto/sql/TestExpressionInterpreter.java | 6 +++++- .../facebook/presto/sql/TestExpressionOptimizer.java | 4 ++-- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 58632aa214e93..098a9cf0979b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -37,6 +37,7 @@ import com.facebook.presto.sql.InterpretedFunctionInvoker; import com.facebook.presto.sql.planner.Interpreters.LambdaSymbolResolver; import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; import com.facebook.presto.util.Failures; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -150,7 +151,7 @@ public Object evaluate() public Object optimize() { checkState(optimize, "optimize() not allowed for interpreter"); - return expression.accept(visitor, null); + return optimize(null); } /** @@ -160,7 +161,13 @@ public Object optimize() public Object optimize(SymbolResolver inputs) { checkState(optimize, "optimize(SymbolResolver) not allowed for interpreter"); - return expression.accept(visitor, inputs); + Object result = expression.accept(visitor, inputs); + + if (!(result instanceof RowExpression)) { + // constant folding + return result; + } + return new ExpressionOptimizer(metadata.getFunctionManager(), session).optimize((RowExpression) result); } private class Visitor diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java index c10cb00110969..bf3860a4d3942 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java @@ -166,7 +166,7 @@ public static RowExpression translate( requireNonNull(result, "translated expression is null"); if (optimize) { - ExpressionOptimizer optimizer = new ExpressionOptimizer(functionManager, session); + ExpressionOptimizer optimizer = new ExpressionOptimizer(functionManager, session.toConnectorSession()); return optimizer.optimize(result); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java index 7310d8ac8747e..c277c4af3dbc5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.sql.relational.optimizer; -import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.spi.ConnectorSession; @@ -62,10 +61,10 @@ public class ExpressionOptimizer private final FunctionManager functionManager; private final ConnectorSession session; - public ExpressionOptimizer(FunctionManager functionManager, Session session) + public ExpressionOptimizer(FunctionManager functionManager, ConnectorSession session) { this.functionManager = functionManager; - this.session = session.toConnectorSession(); + this.session = session; } public RowExpression optimize(RowExpression expression) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index 730514c98d6a3..533201d145d25 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -89,6 +89,7 @@ import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionInterpreter; import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; import static com.facebook.presto.sql.planner.RowExpressionInterpreter.rowExpressionInterpreter; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.util.DateTimeZoneIndex.getDateTimeZone; import static io.airlift.slice.Slices.utf8Slice; @@ -352,6 +353,9 @@ public void testFunctionCall() assertOptimizedEquals("abs(-bound_long)", "1234"); assertOptimizedEquals("abs(unbound_long)", "abs(unbound_long)"); assertOptimizedEquals("abs(unbound_long + 1)", "abs(unbound_long + 1)"); + assertOptimizedEquals("cast(json_parse(unbound_string) as map(varchar, varchar))", "cast(json_parse(unbound_string) as map(varchar, varchar))"); + assertOptimizedEquals("cast(json_parse(unbound_string) as array(varchar))", "cast(json_parse(unbound_string) as array(varchar))"); + assertOptimizedEquals("cast(json_parse(unbound_string) as row(bigint, varchar))", "cast(json_parse(unbound_string) as row(bigint, varchar))"); } @Test @@ -1582,7 +1586,7 @@ private static void assertExpressionAndRowExpressionEquals(Object expressionResu // It is tricky to check the equivalence of an expression and a row expression. // We rely on the optimized translator to fill the gap. RowExpression translated = TRANSLATOR.translateAndOptimize((Expression) expressionResult, SYMBOL_TYPES); - assertRowExpressionEvaluationEquals(translated, new ExpressionOptimizer(METADATA.getFunctionManager(), TEST_SESSION).optimize((RowExpression) rowExpressionResult)); + assertRowExpressionEvaluationEquals(translated, new ExpressionOptimizer(METADATA.getFunctionManager(), SESSION).optimize((RowExpression) rowExpressionResult)); } else { // We have constants; directly compare diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java index fd5ef1dd4d9d5..ec1284645f325 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java @@ -32,7 +32,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.toValues; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.metadata.CastType.JSON_TO_ARRAY_CAST; @@ -50,6 +49,7 @@ import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.field; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.type.JsonType.JSON; import static com.facebook.presto.util.StructuralTestUtil.mapType; import static io.airlift.slice.Slices.utf8Slice; @@ -66,7 +66,7 @@ public void setUp() { TypeManager typeManager = new TypeRegistry(); functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - optimizer = new ExpressionOptimizer(functionManager, TEST_SESSION); + optimizer = new ExpressionOptimizer(functionManager, SESSION); } @AfterClass(alwaysRun = true) From abbac155b0ad4f25af4b1fafcaa9fd2df06b31ac Mon Sep 17 00:00:00 2001 From: James Sun Date: Fri, 19 Apr 2019 12:31:43 -0700 Subject: [PATCH 43/66] Extract bindChannels helper in LocalExecutionPlanner bindChannels is to replace toRowExpression in LocalExecutionPlanner. It generates channel info for expressions as well as optimizes them. --- .../sql/planner/LocalExecutionPlanner.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 175a9ac55e327..b1c5f4dd93f6a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -1205,18 +1205,8 @@ private PhysicalOperation visitScanFilterAndProject( // filterExpression may contain large function calls; evaluate them before compiling. if (filterExpression.isPresent()) { - Type type = filterExpression.get().getType(); - Object value = new RowExpressionInterpreter(filterExpression.get(), metadata, session.toConnectorSession(), true).optimize(); - if (value instanceof RowExpression) { - RowExpression optimizedFilter = (RowExpression) value; - // building channel info - optimizedFilter = SymbolToChannelTranslator.translate(optimizedFilter, sourceLayout); - filterExpression = Optional.of(optimizedFilter); - } - else { - // TODO: theoretically value should either be true or false; we could possibly optimized the execution based on this - filterExpression = Optional.of(constant(value, type)); - } + // TODO: theoretically, filterExpression could be a constant value (true or false) after optimization; we could possibly optimize the execution. + filterExpression = Optional.of(bindChannels(filterExpression.get(), sourceLayout)); } // build output mapping @@ -1286,11 +1276,27 @@ private PhysicalOperation visitScanFilterAndProject( } } + // TODO: migrate `toRowExpression` to `bindChannels` private RowExpression toRowExpression(Expression expression, Map, Type> types, Map sourceLayout) { return SqlToRowExpressionTranslator.translate(expression, types, sourceLayout, metadata.getFunctionManager(), metadata.getTypeManager(), session, true); } + private RowExpression bindChannels(RowExpression expression, Map sourceLayout) + { + Type type = expression.getType(); + Object value = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), true).optimize(); + if (value instanceof RowExpression) { + RowExpression optimized = (RowExpression) value; + // building channel info + expression = SymbolToChannelTranslator.translate(optimized, sourceLayout); + } + else { + expression = constant(value, type); + } + return expression; + } + @Override public PhysicalOperation visitTableScan(TableScanNode node, LocalExecutionPlanContext context) { From 220c0fe1f8bdf066ee6197d5040081c8aa1ac409 Mon Sep 17 00:00:00 2001 From: Leiqing Cai Date: Fri, 19 Apr 2019 02:16:23 -0700 Subject: [PATCH 44/66] Support JdbcErrorCode to Verifier exception classifier Also, mark JDBC_ERROR as retryable. --- presto-verifier/pom.xml | 5 +++++ .../verifier/framework/PrestoExceptionClassifier.java | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index e49117926e4db..9ac5f048fe2f0 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -17,6 +17,11 @@ + + com.facebook.presto + presto-base-jdbc + + com.facebook.presto presto-hive diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java index da896c32f72e1..988507a94a19f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.HiveErrorCode; import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.plugin.jdbc.JdbcErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.StandardErrorCode; import com.google.common.collect.ImmutableSet; @@ -37,6 +38,7 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; +import static com.facebook.presto.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; import static com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_ERROR; import static com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT; @@ -55,6 +57,7 @@ public class PrestoExceptionClassifier private static final Set DEFAULT_ERRORS = ImmutableSet.builder() .addAll(asList(StandardErrorCode.values())) .addAll(asList(HiveErrorCode.values())) + .addAll(asList(JdbcErrorCode.values())) .build(); private static final Set DEFAULT_RETRYABLE_ERRORS = ImmutableSet.of( @@ -75,7 +78,9 @@ public class PrestoExceptionClassifier HIVE_WRITER_CLOSE_ERROR, HIVE_FILESYSTEM_ERROR, HIVE_CANNOT_OPEN_SPLIT, - HIVE_METASTORE_ERROR); + HIVE_METASTORE_ERROR, + // From JdbcErrorCode + JDBC_ERROR); private final Map errorByCode; private final Set retryableErrors; From 87e0cca600579b7596de9c98e242d4a4877608f1 Mon Sep 17 00:00:00 2001 From: James Sun Date: Mon, 8 Apr 2019 00:18:47 -0700 Subject: [PATCH 45/66] Add compression kind config for Raptor --- .../raptor/storage/StorageManagerConfig.java | 15 +++++++++++++++ .../raptor/storage/TestStorageManagerConfig.java | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java index ead02214313c9..fe3d773240737 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.type.TimeZoneKey; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; @@ -33,6 +34,7 @@ import java.util.TimeZone; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -55,6 +57,7 @@ public class StorageManagerConfig private DataSize orcTinyStripeThreshold = new DataSize(8, MEGABYTE); private boolean orcLazyReadSmallRanges = true; private OrcOptimizedWriterStage orcOptimizedWriterStage = DISABLED; + private CompressionKind orcCompressionKind = SNAPPY; private int deletionThreads = max(1, getRuntime().availableProcessors() / 2); private int recoveryThreads = 10; private int organizationThreads = 5; @@ -180,6 +183,18 @@ public StorageManagerConfig setOrcOptimizedWriterStage(OrcOptimizedWriterStage o return this; } + public CompressionKind getOrcCompressionKind() + { + return orcCompressionKind; + } + + @Config("storage.orc.compression-kind") + public StorageManagerConfig setOrcCompressionKind(CompressionKind orcCompressionKind) + { + this.orcCompressionKind = orcCompressionKind; + return this; + } + @Min(1) public int getDeletionThreads() { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java index 52c526a5cbf82..d9dc2e5d5a3f9 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java @@ -24,6 +24,8 @@ import java.io.File; import java.util.Map; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; @@ -55,6 +57,7 @@ public void testDefaults() .setOrcTinyStripeThreshold(new DataSize(8, MEGABYTE)) .setOrcLazyReadSmallRanges(true) .setOrcOptimizedWriterStage(DISABLED) + .setOrcCompressionKind(SNAPPY) .setDeletionThreads(max(1, getRuntime().availableProcessors() / 2)) .setShardRecoveryTimeout(new Duration(30, SECONDS)) .setMissingShardDiscoveryInterval(new Duration(5, MINUTES)) @@ -85,6 +88,7 @@ public void testExplicitPropertyMappings() .put("storage.orc.tiny-stripe-threshold", "15kB") .put("storage.orc.lazy-read-small-ranges", "false") .put("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED") + .put("storage.orc.compression-kind", "ZSTD") .put("storage.max-deletion-threads", "999") .put("storage.shard-recovery-timeout", "1m") .put("storage.missing-shard-discovery-interval", "4m") @@ -112,6 +116,7 @@ public void testExplicitPropertyMappings() .setOrcTinyStripeThreshold(new DataSize(15, KILOBYTE)) .setOrcLazyReadSmallRanges(false) .setOrcOptimizedWriterStage(ENABLED_AND_VALIDATED) + .setOrcCompressionKind(ZSTD) .setDeletionThreads(999) .setShardRecoveryTimeout(new Duration(1, MINUTES)) .setMissingShardDiscoveryInterval(new Duration(4, MINUTES)) From eaf7e7be7febc750edeb7e83cf3e2d139749b51a Mon Sep 17 00:00:00 2001 From: James Sun Date: Mon, 8 Apr 2019 00:58:00 -0700 Subject: [PATCH 46/66] Allow Raptor to compress data with ZSTD --- .../presto/raptor/RaptorErrorCode.java | 3 +- .../presto/raptor/storage/OrcFileWriter.java | 11 +++---- .../raptor/storage/OrcPageFileRewriter.java | 9 ++++-- .../raptor/storage/OrcRecordWriter.java | 30 +++++++++++++++---- .../raptor/storage/OrcStorageManager.java | 11 +++++-- .../presto/raptor/storage/OrcTestingUtil.java | 6 ++-- .../raptor/storage/TestOrcFileRewriter.java | 8 +++-- .../raptor/storage/TestOrcStorageManager.java | 2 ++ 8 files changed, 57 insertions(+), 23 deletions(-) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java index 09da294fac836..ecafb08057adb 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java @@ -37,7 +37,8 @@ public enum RaptorErrorCode RAPTOR_LOCAL_DISK_FULL(12, EXTERNAL), RAPTOR_BACKUP_CORRUPTION(13, EXTERNAL), RAPTOR_NOT_ENOUGH_NODES(14, EXTERNAL), - RAPTOR_WRITER_DATA_ERROR(15, EXTERNAL); + RAPTOR_WRITER_DATA_ERROR(15, EXTERNAL), + RAPTOR_UNSUPPORTED_COMPRESSION_KIND(16, EXTERNAL); private final ErrorCode errorCode; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java index 28332e976a819..c3c735b4306c5 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java @@ -17,6 +17,7 @@ import com.facebook.presto.orc.OrcWriterOptions; import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; @@ -39,7 +40,6 @@ import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; -import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_WRITER_DATA_ERROR; import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; @@ -60,9 +60,9 @@ public class OrcFileWriter private long rowCount; private long uncompressedSize; - public OrcFileWriter(List columnIds, List columnTypes, File target, boolean validate, OrcWriterStats stats, TypeManager typeManager) + public OrcFileWriter(List columnIds, List columnTypes, File target, boolean validate, OrcWriterStats stats, TypeManager typeManager, CompressionKind compression) { - this(columnIds, columnTypes, target, true, validate, stats, typeManager); + this(columnIds, columnTypes, target, true, validate, stats, typeManager, compression); } @VisibleForTesting @@ -73,7 +73,8 @@ public OrcFileWriter(List columnIds, List columnTypes, File target, boolean writeMetadata, boolean validate, OrcWriterStats stats, - TypeManager typeManager) + TypeManager typeManager, + CompressionKind compression) { checkArgument(requireNonNull(columnIds, "columnIds is null").size() == requireNonNull(columnTypes, "columnTypes is null").size(), "ids and types mismatch"); checkArgument(isUnique(columnIds), "ids must be unique"); @@ -99,7 +100,7 @@ public OrcFileWriter(List columnIds, List columnTypes, File target, columnNames, storageTypes, ORC, - SNAPPY, + requireNonNull(compression, "compression is null"), DEFAULT_OPTION, userMetadata, DEFAULT_STORAGE_TIMEZONE, diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java index 5a6ed8c7cc12c..16dab6d41bbc7 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java @@ -19,6 +19,7 @@ import com.facebook.presto.orc.OrcWriter; import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.raptor.util.Closer; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; @@ -48,7 +49,6 @@ import static com.facebook.presto.orc.OrcPredicate.TRUE; import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; -import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.raptor.storage.OrcFileWriter.DEFAULT_OPTION; import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; import static com.facebook.presto.raptor.storage.OrcStorageManager.HUGE_MAX_READ_BLOCK_SIZE; @@ -70,17 +70,20 @@ public final class OrcPageFileRewriter private final boolean validate; private final OrcWriterStats stats; private final TypeManager typeManager; + private final CompressionKind compression; OrcPageFileRewriter( ReaderAttributes readerAttributes, boolean validate, OrcWriterStats stats, - TypeManager typeManager) + TypeManager typeManager, + CompressionKind compression) { this.readerAttributes = requireNonNull(readerAttributes, "readerAttributes is null"); this.validate = validate; this.stats = requireNonNull(stats, "stats is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.compression = requireNonNull(compression, "compression is null"); } @Override @@ -159,7 +162,7 @@ public OrcFileInfo rewrite(Map allColumnTypes, File input, File ou writerColumnIds, writerStorageTypes, ORC, - SNAPPY, + compression, DEFAULT_OPTION, userMetadata, DEFAULT_STORAGE_TIMEZONE, diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java index 34e8930987967..ba565c72d8673 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.raptor.util.SyncingFileSystem; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; @@ -56,6 +57,7 @@ import java.util.Properties; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_UNSUPPORTED_COMPRESSION_KIND; import static com.facebook.presto.raptor.storage.Row.extractRow; import static com.facebook.presto.raptor.storage.StorageType.arrayOf; import static com.facebook.presto.raptor.storage.StorageType.mapOf; @@ -72,7 +74,9 @@ import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES; import static org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; +import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.NONE; import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.SNAPPY; +import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.ZLIB; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.LIST; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.MAP; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE; @@ -101,13 +105,13 @@ public class OrcRecordWriter private long rowCount; private long uncompressedSize; - public OrcRecordWriter(List columnIds, List columnTypes, File target) + public OrcRecordWriter(List columnIds, List columnTypes, File target, CompressionKind compression) { - this(columnIds, columnTypes, target, true); + this(columnIds, columnTypes, target, compression, true); } @VisibleForTesting - OrcRecordWriter(List columnIds, List columnTypes, File target, boolean writeMetadata) + OrcRecordWriter(List columnIds, List columnTypes, File target, CompressionKind compression, boolean writeMetadata) { this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); @@ -122,7 +126,7 @@ public OrcRecordWriter(List columnIds, List columnTypes, File target properties.setProperty(META_TABLE_COLUMN_TYPES, Joiner.on(':').join(hiveTypeNames)); serializer = createSerializer(properties); - recordWriter = createRecordWriter(new Path(target.toURI()), columnIds, columnTypes, writeMetadata); + recordWriter = createRecordWriter(new Path(target.toURI()), columnIds, columnTypes, requireNonNull(compression, "compression is null"), writeMetadata); tableInspector = getStandardStructObjectInspector(columnNames, getJavaObjectInspectors(storageTypes)); structFields = ImmutableList.copyOf(tableInspector.getAllStructFieldRefs()); @@ -197,14 +201,14 @@ private static OrcSerde createSerializer(Properties properties) return serde; } - private static RecordWriter createRecordWriter(Path target, List columnIds, List columnTypes, boolean writeMetadata) + private static RecordWriter createRecordWriter(Path target, List columnIds, List columnTypes, CompressionKind compression, boolean writeMetadata) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { OrcFile.WriterOptions options = new OrcWriterOptions(CONFIGURATION) .memory(new NullMemoryManager(CONFIGURATION)) .fileSystem(fileSystem) - .compress(SNAPPY); + .compress(toCompressionKind(compression)); if (writeMetadata) { options.callback(createFileMetadataCallback(columnIds, columnTypes)); @@ -217,6 +221,20 @@ private static RecordWriter createRecordWriter(Path target, List columnIds } } + private static org.apache.hadoop.hive.ql.io.orc.CompressionKind toCompressionKind(CompressionKind compression) + { + switch (compression) { + case NONE: + return NONE; + case SNAPPY: + return SNAPPY; + case ZLIB: + return ZLIB; + default: + throw new PrestoException(RAPTOR_UNSUPPORTED_COMPRESSION_KIND, "Found unsupported compression kind: " + compression); + } + } + private static OrcFile.WriterCallback createFileMetadataCallback(List columnIds, List columnTypes) { return new OrcFile.WriterCallback() diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index c2c443299057b..866324caabae1 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -22,6 +22,7 @@ import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.orc.TupleDomainOrcPredicate; import com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; +import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.orc.metadata.OrcType; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.RaptorConnectorId; @@ -155,6 +156,7 @@ public class OrcStorageManager private final long maxShardRows; private final DataSize maxShardSize; private final DataSize minAvailableSpace; + private final CompressionKind compression; private final OrcOptimizedWriterStage orcOptimizedWriterStage; private final TypeManager typeManager; private final ExecutorService deletionExecutor; @@ -189,6 +191,7 @@ public OrcStorageManager( config.getMaxShardRows(), config.getMaxShardSize(), config.getMinAvailableSpace(), + config.getOrcCompressionKind(), config.getOrcOptimizedWriterStage()); } @@ -207,6 +210,7 @@ public OrcStorageManager( long maxShardRows, DataSize maxShardSize, DataSize minAvailableSpace, + CompressionKind compression, OrcOptimizedWriterStage orcOptimizedWriterStage) { this.nodeId = requireNonNull(nodeId, "nodeId is null"); @@ -226,9 +230,10 @@ public OrcStorageManager( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.deletionExecutor = newFixedThreadPool(deletionThreads, daemonThreadsNamed("raptor-delete-" + connectorId + "-%s")); this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); + this.compression = requireNonNull(compression, "compression is null"); this.orcOptimizedWriterStage = requireNonNull(orcOptimizedWriterStage, "orcOptimizedWriterStage is null"); if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { - this.fileRewriter = new OrcPageFileRewriter(readerAttributes, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager); + this.fileRewriter = new OrcPageFileRewriter(readerAttributes, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); } else { this.fileRewriter = new OrcRecordFileRewriter(); @@ -711,10 +716,10 @@ private void createWriterIfNecessary() storageService.createParents(stagingFile); stagingFiles.add(stagingFile); if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { - writer = new OrcFileWriter(columnIds, columnTypes, stagingFile, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager); + writer = new OrcFileWriter(columnIds, columnTypes, stagingFile, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); } else { - writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile); + writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile, compression); } } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java index 83b39ab8d1a43..a8d5600e0a8ac 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java @@ -38,6 +38,8 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static org.testng.Assert.assertEquals; @@ -107,8 +109,8 @@ public static FileWriter createFileWriter(List columnIds, List colum if (useOptimizedOrcWriter) { TypeRegistry typeManager = new TypeRegistry(); new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), typeManager); + return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), typeManager, ZSTD); } - return new OrcRecordWriter(columnIds, columnTypes, file, true); + return new OrcRecordWriter(columnIds, columnTypes, file, SNAPPY, true); } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java index dace510727c97..5d28e7fbb6a05 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java @@ -57,6 +57,8 @@ import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.facebook.presto.raptor.storage.FileStorageService.getFileSystemPath; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; @@ -687,9 +689,9 @@ public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) private static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean writeMetadata, boolean useOptimizedOrcWriter) { if (useOptimizedOrcWriter) { - return new OrcFileWriter(columnIds, columnTypes, file, writeMetadata, true, new OrcWriterStats(), new TypeRegistry()); + return new OrcFileWriter(columnIds, columnTypes, file, writeMetadata, true, new OrcWriterStats(), new TypeRegistry(), ZSTD); } - return new OrcRecordWriter(columnIds, columnTypes, file, writeMetadata); + return new OrcRecordWriter(columnIds, columnTypes, file, SNAPPY, writeMetadata); } private static FileRewriter createFileRewriter(boolean useOptimizedOrcWriter) @@ -697,7 +699,7 @@ private static FileRewriter createFileRewriter(boolean useOptimizedOrcWriter) if (useOptimizedOrcWriter) { TypeRegistry typeManager = new TypeRegistry(); new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - return new OrcPageFileRewriter(READER_ATTRIBUTES, true, new OrcWriterStats(), typeManager); + return new OrcPageFileRewriter(READER_ATTRIBUTES, true, new OrcWriterStats(), typeManager, ZSTD); } return new OrcRecordFileRewriter(); } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java index 622b7a4e1e715..0b488544e29cd 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java @@ -71,6 +71,7 @@ import java.util.stream.IntStream; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; import static com.facebook.presto.raptor.storage.OrcStorageManager.xxhash64; @@ -648,6 +649,7 @@ public static OrcStorageManager createOrcStorageManager( maxShardRows, maxFileSize, new DataSize(0, BYTE), + SNAPPY, useOptimizedWriter ? ENABLED_AND_VALIDATED : DISABLED); } From 3bd21608342b20dc67df24bcf09575c9c6937a3b Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Tue, 9 Apr 2019 17:58:24 -0400 Subject: [PATCH 47/66] Change approx_distinct aggregation intermediate type --- ...gregationsWithExchangeMaterialization.java | 12 ---- ...tedQueriesWithExchangeMaterialization.java | 72 ------------------- .../state/HyperLogLogStateSerializer.java | 8 +-- 3 files changed, 4 insertions(+), 88 deletions(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java index 43a3140254a81..8d317e56d0099 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java @@ -46,18 +46,6 @@ public void testGroupByRow() // row type is not supported by the Hive hash code function } - @Override - public void testApproximateCountDistinctGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testApproximateCountDistinctGroupByWithStandardError() - { - // Unsupported Hive type: HyperLogLog - } - @Override public void testAggregationPushedBelowOuterJoin() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java index 2420765c9b5d2..a505372c2deaa 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java @@ -98,78 +98,6 @@ public void testAntiJoinNullHandling() // Unsupported Hive type: unknown } - @Override - public void testApproxSetBigintGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testApproxSetVarcharGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testApproxSetDoubleGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testApproxSetGroupByWithOnlyNullsInOneGroup() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testApproxSetGroupByWithNulls() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testMergeHyperLogLogGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testMergeHyperLogLogGroupByWithNulls() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testP4ApproxSetBigintGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testP4ApproxSetVarcharGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testP4ApproxSetDoubleGroupBy() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testP4ApproxSetGroupByWithOnlyNullsInOneGroup() - { - // Unsupported Hive type: HyperLogLog - } - - @Override - public void testP4ApproxSetGroupByWithNulls() - { - // Unsupported Hive type: HyperLogLog - } - @Override public void testHaving3() { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java index 56c1a8ce6af62..4b900b6074466 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java @@ -19,7 +19,7 @@ import com.facebook.presto.spi.type.Type; import io.airlift.stats.cardinality.HyperLogLog; -import static com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; public class HyperLogLogStateSerializer implements AccumulatorStateSerializer @@ -27,7 +27,7 @@ public class HyperLogLogStateSerializer @Override public Type getSerializedType() { - return HYPER_LOG_LOG; + return VARBINARY; } @Override @@ -37,13 +37,13 @@ public void serialize(HyperLogLogState state, BlockBuilder out) out.appendNull(); } else { - HYPER_LOG_LOG.writeSlice(out, state.getHyperLogLog().serialize()); + VARBINARY.writeSlice(out, state.getHyperLogLog().serialize()); } } @Override public void deserialize(Block block, int index, HyperLogLogState state) { - state.setHyperLogLog(HyperLogLog.newInstance(HYPER_LOG_LOG.getSlice(block, index))); + state.setHyperLogLog(HyperLogLog.newInstance(VARBINARY.getSlice(block, index))); } } From 28e45a9c218967b01f5fa3ff3dc99d0b60f1831b Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Thu, 18 Apr 2019 11:21:18 -0400 Subject: [PATCH 48/66] Change min_by, max_by aggregations intermediate type Avoid using anonymous row type, as this type cannot be serialized by the Hive connector that is used as a temporary table connector for exchange materialization --- ...ggregationsWithExchangeMaterialization.java | 6 ------ ...utedQueriesWithExchangeMaterialization.java | 18 ------------------ .../aggregation/AccumulatorCompiler.java | 2 +- .../InternalAggregationFunction.java | 2 +- .../aggregation/TypedKeyValueHeap.java | 2 +- ...eyAndBlockPositionValueStateSerializer.java | 2 +- .../MultimapAggregationStateSerializer.java | 2 +- .../state/MinMaxNStateSerializer.java | 2 +- .../aggregation/state/StateCompiler.java | 2 +- .../com/facebook/presto/spi/type/RowType.java | 9 +++++++++ 10 files changed, 16 insertions(+), 31 deletions(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java index 8d317e56d0099..31253785f56ad 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedAggregationsWithExchangeMaterialization.java @@ -45,10 +45,4 @@ public void testGroupByRow() { // row type is not supported by the Hive hash code function } - - @Override - public void testAggregationPushedBelowOuterJoin() - { - // Anonymous row type is not supported in Hive - } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java index a505372c2deaa..c668d3c0c76ca 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java @@ -98,24 +98,6 @@ public void testAntiJoinNullHandling() // Unsupported Hive type: unknown } - @Override - public void testHaving3() - { - // Anonymous row type is not supported in Hive - } - - @Override - public void testJoinCoercionOnEqualityComparison() - { - // Anonymous row type is not supported in Hive - } - - @Override - public void testCorrelatedScalarSubqueriesWithScalarAggregation() - { - // Anonymous row type is not supported in Hive - } - @Test public void testExplainOfCreateTableAs() { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java index ab550944e27ed..dd78da3c43d79 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AccumulatorCompiler.java @@ -218,7 +218,7 @@ private static MethodDefinition generateGetIntermediateType(ClassDefinition defi } else { methodDefinition.getBody() - .append(constantType(callSiteBinder, RowType.anonymous(type))) + .append(constantType(callSiteBinder, RowType.withDefaultFieldNames(type))) .retObject(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java index f2485704a23f2..25e0df6d4030b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/InternalAggregationFunction.java @@ -102,7 +102,7 @@ public Type getIntermediateType() return getOnlyElement(intermediateType); } else { - return RowType.anonymous(intermediateType); + return RowType.withDefaultFieldNames(intermediateType); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java index 70ccb03fbbf6d..c062835f83baa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedKeyValueHeap.java @@ -56,7 +56,7 @@ public TypedKeyValueHeap(BlockComparator keyComparator, Type keyType, Type value public static Type getSerializedType(Type keyType, Type valueType) { - return RowType.anonymous(ImmutableList.of(BIGINT, new ArrayType(keyType), new ArrayType(valueType))); + return RowType.withDefaultFieldNames(ImmutableList.of(BIGINT, new ArrayType(keyType), new ArrayType(valueType))); } public int getCapacity() diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java index 22fa53fdaad6f..ef6f9f04f0af1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/KeyAndBlockPositionValueStateSerializer.java @@ -47,7 +47,7 @@ public abstract class KeyAndBlockPositionValueStateSerializer 1) { List types = fields.stream().map(StateField::getSqlType).collect(toImmutableList()); - type = RowType.anonymous(types); + type = RowType.withDefaultFieldNames(types); } else if (fields.size() == 1) { type = getOnlyElement(fields).getSqlType(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java index 498063f219fe4..22055f46f9e81 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java @@ -63,6 +63,15 @@ public static RowType anonymous(List types) return new RowType(makeSignature(fields), fields); } + public static RowType withDefaultFieldNames(List types) + { + List fields = new ArrayList<>(); + for (int i = 0; i < types.size(); i++) { + fields.add(new Field(Optional.of("field" + i), types.get(i))); + } + return new RowType(makeSignature(fields), fields); + } + // Only RowParametricType.createType should call this method public static RowType createWithTypeSignature(TypeSignature typeSignature, List fields) { From aa20c071267ccd3cd0dc6375f44a4f10eb0098d7 Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Fri, 19 Apr 2019 16:53:13 -0400 Subject: [PATCH 49/66] Change qdigest aggregations intermediate type --- .../MergeQuantileDigestFunction.java | 6 +- .../QuantileDigestAggregationFunction.java | 2 +- .../state/QuantileDigestStateSerializer.java | 16 ++---- .../tests/AbstractTestAggregations.java | 56 +++++++++++++++++++ 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java index ce1516177b4b9..efe277f211597 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java @@ -80,13 +80,13 @@ public InternalAggregationFunction specialize(BoundVariables boundVariables, int Type valueType = boundVariables.getTypeVariable("T"); QuantileDigestType outputType = (QuantileDigestType) typeManager.getParameterizedType(StandardTypes.QDIGEST, ImmutableList.of(TypeSignatureParameter.of(valueType.getTypeSignature()))); - return generateAggregation(valueType, outputType); + return generateAggregation(outputType); } - private static InternalAggregationFunction generateAggregation(Type valueType, QuantileDigestType type) + private static InternalAggregationFunction generateAggregation(QuantileDigestType type) { DynamicClassLoader classLoader = new DynamicClassLoader(MapAggregationFunction.class.getClassLoader()); - QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(valueType); + QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(); Type intermediateType = stateSerializer.getSerializedType(); AggregationMetadata metadata = new AggregationMetadata( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java index 72d6d7c1a1ea3..b59abd51dfe58 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java @@ -99,7 +99,7 @@ private static InternalAggregationFunction generateAggregation(Type valueType, Q { DynamicClassLoader classLoader = new DynamicClassLoader(QuantileDigestAggregationFunction.class.getClassLoader()); List inputTypes = getInputTypes(valueType, arity); - QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(valueType); + QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(); Type intermediateType = stateSerializer.getSerializedType(); AggregationMetadata metadata = new AggregationMetadata( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java index e51fb4d60ac4e..aa1d33ea14dd1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java @@ -16,24 +16,18 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; -import com.facebook.presto.spi.type.QuantileDigestType; import com.facebook.presto.spi.type.Type; import io.airlift.stats.QuantileDigest; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; + public class QuantileDigestStateSerializer implements AccumulatorStateSerializer { - private final QuantileDigestType type; - - public QuantileDigestStateSerializer(Type elementType) - { - this.type = new QuantileDigestType(elementType); - } - @Override public Type getSerializedType() { - return type; + return VARBINARY; } @Override @@ -43,13 +37,13 @@ public void serialize(QuantileDigestState state, BlockBuilder out) out.appendNull(); } else { - type.writeSlice(out, state.getQuantileDigest().serialize()); + VARBINARY.writeSlice(out, state.getQuantileDigest().serialize()); } } @Override public void deserialize(Block block, int index, QuantileDigestState state) { - state.setQuantileDigest(new QuantileDigest(type.getSlice(block, index))); + state.setQuantileDigest(new QuantileDigest(VARBINARY.getSlice(block, index))); } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java index e55cc1df6b89e..d2172c1924927 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java @@ -1269,4 +1269,60 @@ public void testOrderedAggregations() "('5-LOW', 445 , NULL)," + "('1-URGENT', 781 , ('O'))"); } + + /** + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction + */ + @Test + public void testQuantileDigest() + { + assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(quantity), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL)), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey, 2), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(quantity, 3), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey, 2, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(quantity, 3, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + } + + /** + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction + */ + @Test + public void testQuantileDigestGroupBy() + { + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey, 2), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity, 3), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey, 2, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity, 3, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + } + + /** + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction + */ + @Test + public void testQuantileDigestMerge() + { + assertQuery("SELECT value_at_quantile(merge(qdigest), 0.5E0) > 0 FROM (SELECT partkey, qdigest_agg(orderkey) as qdigest FROM lineitem GROUP BY partkey)", "SELECT true"); + } + + /** + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction + */ + @Test + public void testQuantileDigestMergeGroupBy() + { + assertQuery("" + + "SELECT partkey, value_at_quantile(merge(qdigest), 0.5E0) > 0 " + + "FROM (SELECT partkey, suppkey, qdigest_agg(orderkey) as qdigest FROM lineitem GROUP BY partkey, suppkey)" + + "GROUP BY partkey", + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + } } From 3e23829f432378a3ec5d3c989fd8d4969875ba6a Mon Sep 17 00:00:00 2001 From: Mayank Garg Date: Tue, 16 Apr 2019 15:49:35 -0700 Subject: [PATCH 50/66] Move presto-server to assembly plugin --- pom.xml | 7 - .../conf/docker/common/compose-commons.sh | 2 +- presto-server/pom.xml | 316 +++++++++++++++++- presto-server/src/main/assembly/presto.xml | 161 +++++++++ .../main/java/com/facebook/presto/Dummy.java | 21 ++ presto-server/src/main/provisio/presto.xml | 196 ----------- 6 files changed, 498 insertions(+), 205 deletions(-) create mode 100644 presto-server/src/main/assembly/presto.xml create mode 100644 presto-server/src/main/java/com/facebook/presto/Dummy.java delete mode 100644 presto-server/src/main/provisio/presto.xml diff --git a/pom.xml b/pom.xml index 37e8da0fcc7fb..2208a292b705a 100644 --- a/pom.xml +++ b/pom.xml @@ -1208,13 +1208,6 @@ true - - io.takari.maven.plugins - provisio-maven-plugin - 0.1.40 - true - - org.apache.maven.plugins maven-compiler-plugin diff --git a/presto-product-tests/conf/docker/common/compose-commons.sh b/presto-product-tests/conf/docker/common/compose-commons.sh index 70c9c247f89c8..ce27ea949c399 100644 --- a/presto-product-tests/conf/docker/common/compose-commons.sh +++ b/presto-product-tests/conf/docker/common/compose-commons.sh @@ -33,7 +33,7 @@ export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"prestodb/hdp2.6-hive"} if [[ -z "${PRESTO_SERVER_DIR:-}" ]]; then source "${PRODUCT_TESTS_ROOT}/target/classes/presto.env" - PRESTO_SERVER_DIR="${PROJECT_ROOT}/presto-server/target/presto-server-${PRESTO_VERSION}/" + PRESTO_SERVER_DIR="${PROJECT_ROOT}/presto-server/target/presto-server-${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}/" fi export_canonical_path PRESTO_SERVER_DIR diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 889629d26ad70..d170d418cbf75 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -10,7 +10,6 @@ presto-server presto-server - provisio ${project.parent.basedir} @@ -24,4 +23,319 @@ com.facebook.presto.server.PrestoServer ${project.artifactId} + + + + + com.facebook.presto + presto-main + + + + + + io.airlift + launcher + ${dep.packaging.version} + bin + tar.gz + provided + + + + io.airlift + launcher + ${dep.packaging.version} + properties + tar.gz + provided + + + + + com.facebook.presto + presto-resource-group-managers + ${project.version} + zip + provided + + + + com.facebook.presto + presto-password-authenticators + ${project.version} + zip + provided + + + + com.facebook.presto + presto-session-property-managers + ${project.version} + zip + provided + + + + com.facebook.presto + presto-jmx + ${project.version} + zip + provided + + + + com.facebook.presto + presto-cassandra + ${project.version} + zip + provided + + + + com.facebook.presto + presto-example-http + ${project.version} + zip + provided + + + + com.facebook.presto + presto-hive-hadoop2 + ${project.version} + zip + provided + + + + com.facebook.presto + presto-memory + ${project.version} + zip + provided + + + + com.facebook.presto + presto-blackhole + ${project.version} + zip + provided + + + + com.facebook.presto + presto-kafka + ${project.version} + zip + provided + + + + com.facebook.presto + presto-kudu + ${project.version} + zip + provided + + + + com.facebook.presto + presto-atop + ${project.version} + zip + provided + + + + com.facebook.presto + presto-ml + ${project.version} + zip + provided + + + + com.facebook.presto + presto-geospatial + ${project.version} + zip + provided + + + + com.facebook.presto + presto-mysql + ${project.version} + zip + provided + + + + com.facebook.presto + presto-postgresql + ${project.version} + zip + provided + + + + com.facebook.presto + presto-redshift + ${project.version} + zip + provided + + + + com.facebook.presto + presto-sqlserver + ${project.version} + zip + provided + + + + com.facebook.presto + presto-raptor + ${project.version} + zip + provided + + + + com.facebook.presto + presto-redis + ${project.version} + zip + provided + + + + com.facebook.presto + presto-tpch + ${project.version} + zip + provided + + + + com.facebook.presto + presto-tpcds + ${project.version} + zip + provided + + + + com.facebook.presto + presto-teradata-functions + ${project.version} + zip + provided + + + + com.facebook.presto + presto-mongodb + ${project.version} + zip + provided + + + + com.facebook.presto + presto-local-file + ${project.version} + zip + provided + + + + com.facebook.presto + presto-accumulo + ${project.version} + zip + provided + + + + com.facebook.presto + presto-thrift-connector + ${project.version} + zip + provided + + + + com.facebook.presto + presto-elasticsearch + ${project.version} + zip + provided + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + bin + package + + single + + + + dir + tar.gz + + + src/main/assembly/presto.xml + + presto-server-${project.version} + false + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-plugins + prepare-package + + unpack-dependencies + + + false + + zip + provided + + + + unpack-launcher + prepare-package + + unpack-dependencies + + + false + launcher + provided + ${project.build.directory}/dependency/launcher + + + + + + diff --git a/presto-server/src/main/assembly/presto.xml b/presto-server/src/main/assembly/presto.xml new file mode 100644 index 0000000000000..01faf6ffef225 --- /dev/null +++ b/presto-server/src/main/assembly/presto.xml @@ -0,0 +1,161 @@ + + presto-server + true + + + + README.txt + + + NOTICE + + + + + + + false + runtime + lib + true + + + + + + + ${project.build.directory}/dependency/launcher/bin + + launcher + launcher.py + + bin + true + + + ${project.build.directory}/dependency/launcher/bin + + launcher + launcher.py + + bin + true + 0755 + + + + + ${project.build.directory}/dependency/presto-resource-group-managers-${project.version} + plugin/resource-group-managers + + + ${project.build.directory}/dependency/presto-password-authenticators-${project.version} + plugin/password-authenticators + + + ${project.build.directory}/dependency/presto-session-property-managers-${project.version} + plugin/session-property-managers + + + ${project.build.directory}/dependency/presto-jmx-${project.version} + plugin/jmx + + + ${project.build.directory}/dependency/presto-cassandra-${project.version} + plugin/cassandra + + + ${project.build.directory}/dependency/presto-example-http-${project.version} + plugin/example-http + + + ${project.build.directory}/dependency/presto-hive-hadoop2-${project.version} + plugin/hive-hadoop2 + + + ${project.build.directory}/dependency/presto-memory-${project.version} + plugin/memory + + + ${project.build.directory}/dependency/presto-blackhole-${project.version} + plugin/blackhole + + + ${project.build.directory}/dependency/presto-kafka-${project.version} + plugin/kafka + + + ${project.build.directory}/dependency/presto-kudu-${project.version} + plugin/kudu + + + ${project.build.directory}/dependency/presto-atop-${project.version} + plugin/atop + + + ${project.build.directory}/dependency/presto-ml-${project.version} + plugin/ml + + + ${project.build.directory}/dependency/presto-geospatial-${project.version} + plugin/geospatial + + + ${project.build.directory}/dependency/presto-mysql-${project.version} + plugin/mysql + + + ${project.build.directory}/dependency/presto-postgresql-${project.version} + plugin/postgresql + + + ${project.build.directory}/dependency/presto-redshift-${project.version} + plugin/redshift + + + ${project.build.directory}/dependency/presto-sqlserver-${project.version} + plugin/sqlserver + + + ${project.build.directory}/dependency/presto-raptor-${project.version} + plugin/raptor + + + ${project.build.directory}/dependency/presto-redis-${project.version} + plugin/redis + + + ${project.build.directory}/dependency/presto-tpch-${project.version} + plugin/tpch + + + ${project.build.directory}/dependency/presto-tpcds-${project.version} + plugin/tpcds + + + ${project.build.directory}/dependency/presto-teradata-functions-${project.version} + plugin/teradata-functions + + + ${project.build.directory}/dependency/presto-mongodb-${project.version} + plugin/mongodb + + + ${project.build.directory}/dependency/presto-local-file-${project.version} + plugin/localfile + + + ${project.build.directory}/dependency/presto-accumulo-${project.version} + plugin/accumulo + + + ${project.build.directory}/dependency/presto-thrift-connector-${project.version} + plugin/presto-thrift + + + ${project.build.directory}/dependency/presto-elasticsearch-${project.version} + plugin/presto-elasticsearch + + + diff --git a/presto-server/src/main/java/com/facebook/presto/Dummy.java b/presto-server/src/main/java/com/facebook/presto/Dummy.java new file mode 100644 index 0000000000000..363367c5c9f66 --- /dev/null +++ b/presto-server/src/main/java/com/facebook/presto/Dummy.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto; + +/** + * This class exists to force the creation of a jar for the presto-server module. This is needed to deploy the presto-server module to nexus. + */ +public class Dummy +{ +} diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml deleted file mode 100644 index a9f95e7c45c5a..0000000000000 --- a/presto-server/src/main/provisio/presto.xml +++ /dev/null @@ -1,196 +0,0 @@ - - - - - - - - NOTICE - README.txt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 2d900d05384b7463e13434134e0803a67719deb5 Mon Sep 17 00:00:00 2001 From: Mayank Garg Date: Thu, 18 Apr 2019 22:00:35 -0700 Subject: [PATCH 51/66] Use nexus-staging-maven-plugin --- pom.xml | 23 ++++++++++ presto-server/pom.xml | 50 +++++++++++----------- presto-server/src/main/assembly/presto.xml | 9 +++- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/pom.xml b/pom.xml index 2208a292b705a..5ce9cbf550405 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,7 @@ 3.8.0 1.2.3 1.10.0 + 1.6.8