diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java index dbad0928beaa6..398e4bac9efcf 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoricalPlanStatisticsUtil.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntry; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntryInfo; import com.facebook.presto.spi.statistics.PlanStatistics; import java.util.ArrayList; @@ -29,26 +30,26 @@ public class HistoricalPlanStatisticsUtil private HistoricalPlanStatisticsUtil() {} /** - * Returns predicted plan statistics depending on historical runs + * Returns historical plan statistics entry containing predicted plan statistics depending on historical runs */ - public static PlanStatistics getPredictedPlanStatistics( + public static Optional getSelectedHistoricalPlanStatisticsEntry( HistoricalPlanStatistics historicalPlanStatistics, List inputTableStatistics, double historyMatchingThreshold) { List lastRunsStatistics = historicalPlanStatistics.getLastRunsStatistics(); if (lastRunsStatistics.isEmpty()) { - return PlanStatistics.empty(); + return Optional.empty(); } Optional similarStatsIndex = getSimilarStatsIndex(historicalPlanStatistics, inputTableStatistics, historyMatchingThreshold); if (similarStatsIndex.isPresent()) { - return lastRunsStatistics.get(similarStatsIndex.get()).getPlanStatistics(); + return Optional.of(lastRunsStatistics.get(similarStatsIndex.get())); } // TODO: Use linear regression to predict stats if we have only 1 table. - return PlanStatistics.empty(); + return Optional.empty(); } /** @@ -58,7 +59,8 @@ public static HistoricalPlanStatistics updatePlanStatistics( HistoricalPlanStatistics historicalPlanStatistics, List inputTableStatistics, PlanStatistics current, - HistoryBasedOptimizationConfig config) + HistoryBasedOptimizationConfig config, + HistoricalPlanStatisticsEntryInfo historicalPlanStatisticsEntryInfo) { List lastRunsStatistics = historicalPlanStatistics.getLastRunsStatistics(); @@ -69,7 +71,7 @@ public static HistoricalPlanStatistics updatePlanStatistics( newLastRunsStatistics.remove(similarStatsIndex.get().intValue()); } - newLastRunsStatistics.add(new HistoricalPlanStatisticsEntry(current, inputTableStatistics)); + newLastRunsStatistics.add(new HistoricalPlanStatisticsEntry(current, inputTableStatistics, historicalPlanStatisticsEntryInfo)); int maxLastRuns = inputTableStatistics.isEmpty() ? 1 : config.getMaxLastRunsHistory(); if (newLastRunsStatistics.size() > maxLastRuns) { newLastRunsStatistics.remove(0); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java index fcdcec906cda9..18f0f7a4fb8a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeWithHash; import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntry; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.statistics.HistoryBasedSourceInfo; import com.facebook.presto.spi.statistics.PlanStatistics; @@ -41,7 +42,7 @@ import static com.facebook.presto.common.RuntimeMetricName.HISTORY_OPTIMIZER_QUERY_REGISTRATION_GET_PLAN_NODE_HASHES; import static com.facebook.presto.common.RuntimeMetricName.HISTORY_OPTIMIZER_QUERY_REGISTRATION_GET_STATISTICS; import static com.facebook.presto.common.RuntimeUnit.NANO; -import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.getPredictedPlanStatistics; +import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.getSelectedHistoricalPlanStatisticsEntry; import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.sql.planner.iterative.Plans.resolveGroupReferences; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -194,11 +195,14 @@ private PlanNodeStatsEstimate getStatistics(PlanNode planNode, Session session, if (allHashes.containsKey(strategy) && entry.getKey().getHash().isPresent() && allHashes.get(strategy).equals(entry.getKey())) { Optional> inputTableStatistics = getPlanNodeInputTableStatistics(plan, session, true); if (inputTableStatistics.isPresent()) { - PlanStatistics predictedPlanStatistics = getPredictedPlanStatistics(entry.getValue(), inputTableStatistics.get(), historyMatchingThreshold); - if (predictedPlanStatistics.getConfidence() > 0) { - return delegateStats.combineStats( - predictedPlanStatistics, - new HistoryBasedSourceInfo(entry.getKey().getHash(), inputTableStatistics)); + Optional historicalPlanStatisticsEntry = getSelectedHistoricalPlanStatisticsEntry(entry.getValue(), inputTableStatistics.get(), historyMatchingThreshold); + if (historicalPlanStatisticsEntry.isPresent()) { + PlanStatistics predictedPlanStatistics = historicalPlanStatisticsEntry.get().getPlanStatistics(); + if (predictedPlanStatistics.getConfidence() > 0) { + return delegateStats.combineStats( + predictedPlanStatistics, + new HistoryBasedSourceInfo(entry.getKey().getHash(), inputTableStatistics, Optional.of(historicalPlanStatisticsEntry.get().getHistoricalPlanStatisticsEntryInfo()))); + } } } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java index 8e68fdfadfe5b..4c9639397be93 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java @@ -14,11 +14,13 @@ package com.facebook.presto.cost; import com.facebook.presto.Session; +import com.facebook.presto.client.NodeVersion; import com.facebook.presto.common.plan.PlanCanonicalizationStrategy; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.spi.statistics.EmptyPlanStatisticsProvider; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.CachingPlanCanonicalInfoProvider; import com.facebook.presto.sql.planner.PlanCanonicalInfoProvider; import com.fasterxml.jackson.databind.ObjectMapper; @@ -39,9 +41,12 @@ public class HistoryBasedPlanStatisticsManager private HistoryBasedPlanStatisticsProvider historyBasedPlanStatisticsProvider = EmptyPlanStatisticsProvider.getInstance(); private boolean statisticsProviderAdded; + private final boolean isNativeExecution; + private final String serverVersion; @Inject - public HistoryBasedPlanStatisticsManager(ObjectMapper objectMapper, SessionPropertyManager sessionPropertyManager, Metadata metadata, HistoryBasedOptimizationConfig config) + public HistoryBasedPlanStatisticsManager(ObjectMapper objectMapper, SessionPropertyManager sessionPropertyManager, Metadata metadata, HistoryBasedOptimizationConfig config, + FeaturesConfig featuresConfig, NodeVersion nodeVersion) { requireNonNull(objectMapper, "objectMapper is null"); this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null"); @@ -49,6 +54,8 @@ public HistoryBasedPlanStatisticsManager(ObjectMapper objectMapper, SessionPrope ObjectMapper newObjectMapper = objectMapper.copy().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); this.planCanonicalInfoProvider = new CachingPlanCanonicalInfoProvider(historyBasedStatisticsCacheManager, newObjectMapper, metadata); this.config = requireNonNull(config, "config is null"); + this.isNativeExecution = featuresConfig.isNativeExecutionEnabled(); + this.serverVersion = requireNonNull(nodeVersion, "nodeVersion is null").toString(); } public void addHistoryBasedPlanStatisticsProviderFactory(HistoryBasedPlanStatisticsProvider historyBasedPlanStatisticsProvider) @@ -67,7 +74,7 @@ public HistoryBasedPlanStatisticsCalculator getHistoryBasedPlanStatisticsCalcula public HistoryBasedPlanStatisticsTracker getHistoryBasedPlanStatisticsTracker() { - return new HistoryBasedPlanStatisticsTracker(() -> historyBasedPlanStatisticsProvider, historyBasedStatisticsCacheManager, sessionPropertyManager, config); + return new HistoryBasedPlanStatisticsTracker(() -> historyBasedPlanStatisticsProvider, historyBasedStatisticsCacheManager, sessionPropertyManager, config, isNativeExecution, serverVersion); } public PlanCanonicalInfoProvider getPlanCanonicalInfoProvider() diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java index 1e92e4b994a6f..3a25cee403228 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntryInfo; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.statistics.HistoryBasedSourceInfo; import com.facebook.presto.spi.statistics.JoinNodeStatistics; @@ -78,17 +79,23 @@ public class HistoryBasedPlanStatisticsTracker private final HistoryBasedStatisticsCacheManager historyBasedStatisticsCacheManager; private final SessionPropertyManager sessionPropertyManager; private final HistoryBasedOptimizationConfig config; + private final boolean isNativeExecution; + private final String serverVersion; public HistoryBasedPlanStatisticsTracker( Supplier historyBasedPlanStatisticsProvider, HistoryBasedStatisticsCacheManager historyBasedStatisticsCacheManager, SessionPropertyManager sessionPropertyManager, - HistoryBasedOptimizationConfig config) + HistoryBasedOptimizationConfig config, + boolean isNativeExecution, + String serverVersion) { this.historyBasedPlanStatisticsProvider = requireNonNull(historyBasedPlanStatisticsProvider, "historyBasedPlanStatisticsProvider is null"); this.historyBasedStatisticsCacheManager = requireNonNull(historyBasedStatisticsCacheManager, "historyBasedStatisticsCacheManager is null"); this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null"); this.config = requireNonNull(config, "config is null"); + this.isNativeExecution = isNativeExecution; + this.serverVersion = serverVersion; } public void updateStatistics(QueryExecution queryExecution) @@ -139,6 +146,9 @@ else if (trackStatsForFailedQueries) { return ImmutableMap.of(); } + HistoricalPlanStatisticsEntryInfo historicalPlanStatisticsEntryInfo = new HistoricalPlanStatisticsEntryInfo( + isNativeExecution ? HistoricalPlanStatisticsEntryInfo.WorkerType.CPP : HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, queryInfo.getQueryId(), serverVersion); + Map planNodeStatsMap = aggregateStageStats(allStages); Map planStatisticsMap = new HashMap<>(); Map canonicalInfoMap = new HashMap<>(); @@ -216,7 +226,7 @@ else if (trackStatsForFailedQueries) { PlanStatisticsWithSourceInfo planStatsWithSourceInfo = new PlanStatisticsWithSourceInfo( planNode.getId(), newPlanNodeStats, - new HistoryBasedSourceInfo(Optional.of(hash), Optional.of(inputTableStatistics))); + new HistoryBasedSourceInfo(Optional.of(hash), Optional.of(inputTableStatistics), Optional.of(historicalPlanStatisticsEntryInfo))); planStatisticsMap.put(planNodeWithHash, planStatsWithSourceInfo); if (isAggregation(planNode, AggregationNode.Step.FINAL) && ((AggregationNode) planNode).getAggregationId().isPresent() && trackPartialAggregationHistory(session)) { @@ -317,7 +327,8 @@ public void updateStatistics(QueryInfo queryInfo) Map newPlanStatistics = planStatistics.entrySet().stream() .filter(entry -> entry.getKey().getHash().isPresent() && entry.getValue().getSourceInfo() instanceof HistoryBasedSourceInfo && - ((HistoryBasedSourceInfo) entry.getValue().getSourceInfo()).getInputTableStatistics().isPresent()) + ((HistoryBasedSourceInfo) entry.getValue().getSourceInfo()).getInputTableStatistics().isPresent() && + ((HistoryBasedSourceInfo) entry.getValue().getSourceInfo()).getHistoricalPlanStatisticsEntryInfo().isPresent()) .collect(toImmutableMap( Map.Entry::getKey, entry -> { @@ -328,7 +339,8 @@ public void updateStatistics(QueryInfo queryInfo) historicalPlanStatistics, historyBasedSourceInfo.getInputTableStatistics().get(), entry.getValue().getPlanStatistics(), - config); + config, + historyBasedSourceInfo.getHistoricalPlanStatisticsEntryInfo().get()); })); if (!newPlanStatistics.isEmpty()) { diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 915c7b855c6bf..3de125e2659e5 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -437,7 +437,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.statsNormalizer = new StatsNormalizer(); this.scalarStatsCalculator = new ScalarStatsCalculator(metadata); this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer); - this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, new SessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig()); + this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, new SessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig(), featuresConfig, new NodeVersion("1")); this.fragmentStatsProvider = new FragmentStatsProvider(); this.statsCalculator = createNewStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer, filterStatsCalculator, historyBasedPlanStatisticsManager, fragmentStatsProvider); this.taskCountEstimator = new TaskCountEstimator(() -> nodeCountForStats); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java index a64923a883c45..428a81524272f 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoricalPlanStatistics.java @@ -13,8 +13,11 @@ */ package com.facebook.presto.cost; +import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntry; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntryInfo; import com.facebook.presto.spi.statistics.JoinNodeStatistics; import com.facebook.presto.spi.statistics.PartialAggregationStatistics; import com.facebook.presto.spi.statistics.PlanStatistics; @@ -23,7 +26,9 @@ import org.testng.annotations.Test; import java.util.List; +import java.util.Optional; +import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.getSelectedHistoricalPlanStatisticsEntry; import static org.testng.Assert.assertEquals; public class TestHistoricalPlanStatistics @@ -97,16 +102,18 @@ private static HistoricalPlanStatistics updatePlanStatistics( historicalPlanStatistics, inputTableStatistics, current, - new HistoryBasedOptimizationConfig()); + new HistoryBasedOptimizationConfig(), + new HistoricalPlanStatisticsEntryInfo(HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, QueryId.valueOf("0"), "test")); } private static PlanStatistics getPredictedPlanStatistics( HistoricalPlanStatistics historicalPlanStatistics, List inputTableStatistics) { - return HistoricalPlanStatisticsUtil.getPredictedPlanStatistics( + Optional historicalPlanStatisticsEntry = getSelectedHistoricalPlanStatisticsEntry( historicalPlanStatistics, inputTableStatistics, 0.1); + return historicalPlanStatisticsEntry.isPresent() ? historicalPlanStatisticsEntry.get().getPlanStatistics() : PlanStatistics.empty(); } } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java index 1f595ea74ce91..02c5ee3324cf8 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedStatsProvider.java @@ -14,12 +14,14 @@ package com.facebook.presto.cost; import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.PlanNodeWithHash; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntry; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntryInfo; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.statistics.JoinNodeStatistics; import com.facebook.presto.spi.statistics.PartialAggregationStatistics; @@ -126,7 +128,8 @@ public Map getStats(List pb.remoteSource(ImmutableList.of(new PlanFragmentId(1)), statsEquivalentRemoteSource)) .check(check -> check.totalSize(1000) @@ -178,7 +179,7 @@ public void testUsesRuntimeStatsWhenHboDisabled() ImmutableList.of( new HistoricalPlanStatisticsEntry( new PlanStatistics(Estimate.of(100), Estimate.of(1000), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()), - ImmutableList.of()))))); + ImmutableList.of(), new HistoricalPlanStatisticsEntryInfo(HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, QueryId.valueOf("0"), "test")))))); tester.assertStatsFor(pb -> pb.remoteSource(ImmutableList.of(new PlanFragmentId(1)))) .check(check -> check.totalSize(1000) @@ -205,7 +206,7 @@ public void testUsesRuntimeStatsWhenDiffersFromHbo() ImmutableList.of( new HistoricalPlanStatisticsEntry( new PlanStatistics(Estimate.of(10), Estimate.of(100), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()), - ImmutableList.of()))))); + ImmutableList.of(), new HistoricalPlanStatisticsEntryInfo(HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, QueryId.valueOf("0"), "test")))))); tester.assertStatsFor(pb -> pb.remoteSource(ImmutableList.of(new PlanFragmentId(1)))) .check(check -> check.totalSize(1000) diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntry.java b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntry.java index 2835019131f56..b37bc8c208f19 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntry.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntry.java @@ -31,13 +31,15 @@ public class HistoricalPlanStatisticsEntry private final PlanStatistics planStatistics; // Size of input tables when plan statistics was recorded. This list will be sorted by input tables canonical order. private final List inputTableStatistics; + private final HistoricalPlanStatisticsEntryInfo historicalPlanStatisticsEntryInfo; @ThriftConstructor - public HistoricalPlanStatisticsEntry(PlanStatistics planStatistics, List inputTableStatistics) + public HistoricalPlanStatisticsEntry(PlanStatistics planStatistics, List inputTableStatistics, HistoricalPlanStatisticsEntryInfo historicalPlanStatisticsEntryInfo) { // Check for nulls, to make it thrift backwards compatible this.planStatistics = planStatistics == null ? PlanStatistics.empty() : planStatistics; this.inputTableStatistics = unmodifiableList(inputTableStatistics == null ? emptyList() : inputTableStatistics); + this.historicalPlanStatisticsEntryInfo = historicalPlanStatisticsEntryInfo; } @ThriftField(value = 1, requiredness = OPTIONAL) @@ -52,6 +54,12 @@ public List getInputTableStatistics() return inputTableStatistics; } + @ThriftField(value = 3, requiredness = OPTIONAL) + public HistoricalPlanStatisticsEntryInfo getHistoricalPlanStatisticsEntryInfo() + { + return historicalPlanStatisticsEntryInfo; + } + @Override public boolean equals(Object o) { @@ -62,18 +70,18 @@ public boolean equals(Object o) return false; } HistoricalPlanStatisticsEntry that = (HistoricalPlanStatisticsEntry) o; - return Objects.equals(planStatistics, that.planStatistics) && Objects.equals(inputTableStatistics, that.inputTableStatistics); + return Objects.equals(planStatistics, that.planStatistics) && Objects.equals(inputTableStatistics, that.inputTableStatistics) && Objects.equals(historicalPlanStatisticsEntryInfo, that.historicalPlanStatisticsEntryInfo); } @Override public int hashCode() { - return Objects.hash(planStatistics, inputTableStatistics); + return Objects.hash(planStatistics, inputTableStatistics, historicalPlanStatisticsEntryInfo); } @Override public String toString() { - return format("HistoricalPlanStatisticsEntry{planStatistics=%s, inputTableStatistics=%s}", planStatistics, inputTableStatistics); + return format("HistoricalPlanStatisticsEntry{planStatistics=%s, inputTableStatistics=%s, historicalPlanStatisticsEntryInfo=%s}", planStatistics, inputTableStatistics, historicalPlanStatisticsEntryInfo); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntryInfo.java b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntryInfo.java new file mode 100644 index 0000000000000..69faaf079b309 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoricalPlanStatisticsEntryInfo.java @@ -0,0 +1,110 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.statistics; + +import com.facebook.drift.annotations.ThriftConstructor; +import com.facebook.drift.annotations.ThriftEnum; +import com.facebook.drift.annotations.ThriftEnumValue; +import com.facebook.drift.annotations.ThriftField; +import com.facebook.drift.annotations.ThriftStruct; +import com.facebook.presto.spi.QueryId; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.facebook.drift.annotations.ThriftField.Requiredness.OPTIONAL; +import static java.lang.String.format; + +@ThriftStruct +public class HistoricalPlanStatisticsEntryInfo +{ + private final WorkerType workerType; + private final QueryId queryId; + private final String serverVersion; + + @JsonCreator + @ThriftConstructor + public HistoricalPlanStatisticsEntryInfo(@JsonProperty("workerType") WorkerType workerType, @JsonProperty("queryId") QueryId queryId, @JsonProperty("serverVersion") String serverVersion) + { + this.workerType = workerType; + this.queryId = queryId; + this.serverVersion = serverVersion; + } + + @JsonProperty + @ThriftField(value = 1, requiredness = OPTIONAL) + public WorkerType getWorkerType() + { + return workerType; + } + + @JsonProperty + @ThriftField(value = 2, requiredness = OPTIONAL) + public QueryId getQueryId() + { + return queryId; + } + + @JsonProperty + @ThriftField(value = 3, requiredness = OPTIONAL) + public String getServerVersion() + { + return serverVersion; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HistoricalPlanStatisticsEntryInfo that = (HistoricalPlanStatisticsEntryInfo) o; + return Objects.equals(workerType, that.workerType) && Objects.equals(queryId, that.queryId) && Objects.equals(serverVersion, that.serverVersion); + } + + @Override + public int hashCode() + { + return Objects.hash(workerType, queryId, serverVersion); + } + + @Override + public String toString() + { + return format("HistoricalPlanStatisticsEntryInfo{workerType=%s, queryId=%s, serverVersion=%s}", workerType, queryId, serverVersion); + } + + @ThriftEnum + public enum WorkerType + { + JAVA(1), CPP(2); + + private final int value; + + WorkerType(int value) + { + this.value = value; + } + + @ThriftEnumValue + public int getValue() + { + return value; + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java index b74a12a1c0ff2..d66cabfb1b3dc 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java @@ -27,11 +27,13 @@ public class HistoryBasedSourceInfo { private final Optional hash; private final Optional> inputTableStatistics; + private final Optional historicalPlanStatisticsEntryInfo; - public HistoryBasedSourceInfo(Optional hash, Optional> inputTableStatistics) + public HistoryBasedSourceInfo(Optional hash, Optional> inputTableStatistics, Optional historicalPlanStatisticsEntryInfo) { this.hash = requireNonNull(hash, "hash is null"); this.inputTableStatistics = requireNonNull(inputTableStatistics, "inputTableStatistics is null"); + this.historicalPlanStatisticsEntryInfo = requireNonNull(historicalPlanStatisticsEntryInfo, "historicalPlanStatisticsEntryInfo is null"); } public Optional getHash() @@ -44,6 +46,11 @@ public Optional> getInputTableStatistics() return inputTableStatistics; } + public Optional getHistoricalPlanStatisticsEntryInfo() + { + return historicalPlanStatisticsEntryInfo; + } + @Override public boolean equals(Object o) { @@ -54,13 +61,13 @@ public boolean equals(Object o) return false; } HistoryBasedSourceInfo that = (HistoryBasedSourceInfo) o; - return Objects.equals(hash, that.hash) && Objects.equals(inputTableStatistics, that.inputTableStatistics); + return Objects.equals(hash, that.hash) && Objects.equals(inputTableStatistics, that.inputTableStatistics) && Objects.equals(historicalPlanStatisticsEntryInfo, that.historicalPlanStatisticsEntryInfo); } @Override public int hashCode() { - return Objects.hash(hash, inputTableStatistics); + return Objects.hash(hash, inputTableStatistics, historicalPlanStatisticsEntryInfo); } @Override diff --git a/redis-hbo-provider/src/test/java/com/facebook/presto/statistic/TestHistoricalStatisticsSerde.java b/redis-hbo-provider/src/test/java/com/facebook/presto/statistic/TestHistoricalStatisticsSerde.java index 8d60f723ff389..66f5714a779ba 100644 --- a/redis-hbo-provider/src/test/java/com/facebook/presto/statistic/TestHistoricalStatisticsSerde.java +++ b/redis-hbo-provider/src/test/java/com/facebook/presto/statistic/TestHistoricalStatisticsSerde.java @@ -13,9 +13,11 @@ */ package com.facebook.presto.statistic; +import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntry; +import com.facebook.presto.spi.statistics.HistoricalPlanStatisticsEntryInfo; import com.facebook.presto.spi.statistics.JoinNodeStatistics; import com.facebook.presto.spi.statistics.PartialAggregationStatistics; import com.facebook.presto.spi.statistics.PlanStatistics; @@ -39,7 +41,8 @@ public void testSimpleHistoricalStatisticsEncoderDecoder() { HistoricalPlanStatistics samplePlanStatistics = new HistoricalPlanStatistics(ImmutableList.of(new HistoricalPlanStatisticsEntry( new PlanStatistics(Estimate.of(100), Estimate.of(1000), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()), - ImmutableList.of(new PlanStatistics(Estimate.of(15000), Estimate.unknown(), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()))))); + ImmutableList.of(new PlanStatistics(Estimate.of(15000), Estimate.unknown(), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())), + new HistoricalPlanStatisticsEntryInfo(HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, QueryId.valueOf("0"), "test")))); HistoricalStatisticsSerde historicalStatisticsEncoderDecoder = new HistoricalStatisticsSerde(); // Test PlanHash @@ -57,7 +60,8 @@ public void testHistoricalPlanStatisticsEntryList() List historicalPlanStatisticsEntryList = new ArrayList<>(); for (int i = 0; i < 50; i++) { historicalPlanStatisticsEntryList.add(new HistoricalPlanStatisticsEntry(new PlanStatistics(Estimate.of(i * 5), Estimate.of(i * 5), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()), - ImmutableList.of(new PlanStatistics(Estimate.of(100), Estimate.of(i), 0, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())))); + ImmutableList.of(new PlanStatistics(Estimate.of(100), Estimate.of(i), 0, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty())), + new HistoricalPlanStatisticsEntryInfo(HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, QueryId.valueOf("0"), "test"))); } HistoricalPlanStatistics samplePlanStatistics = new HistoricalPlanStatistics(historicalPlanStatisticsEntryList); HistoricalStatisticsSerde historicalStatisticsEncoderDecoder = new HistoricalStatisticsSerde(); @@ -88,7 +92,7 @@ public void testPlanStatisticsList() List historicalPlanStatisticsEntryList = new ArrayList<>(); for (int i = 0; i < 50; i++) { historicalPlanStatisticsEntryList.add(new HistoricalPlanStatisticsEntry(new PlanStatistics(Estimate.of(i * 5), Estimate.of(i * 5), 1, JoinNodeStatistics.empty(), TableWriterNodeStatistics.empty(), PartialAggregationStatistics.empty()), - planStatisticsEntryList)); + planStatisticsEntryList, new HistoricalPlanStatisticsEntryInfo(HistoricalPlanStatisticsEntryInfo.WorkerType.JAVA, QueryId.valueOf("0"), "test"))); } HistoricalPlanStatistics samplePlanStatistics = new HistoricalPlanStatistics(historicalPlanStatisticsEntryList); HistoricalStatisticsSerde historicalStatisticsEncoderDecoder = new HistoricalStatisticsSerde();