diff --git a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java index 2953460c3a461..80c633b88f132 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -261,6 +261,7 @@ public final class SystemSessionProperties public static final String TRACK_HISTORY_STATS_FROM_FAILED_QUERIES = "track_history_stats_from_failed_queries"; public static final String USE_PERFECTLY_CONSISTENT_HISTORIES = "use_perfectly_consistent_histories"; public static final String HISTORY_CANONICAL_PLAN_NODE_LIMIT = "history_canonical_plan_node_limit"; + public static final String HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES = "history_based_optimizer_estimate_size_using_variables"; public static final String HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT = "history_based_optimizer_timeout_limit"; public static final String RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY = "restrict_history_based_optimization_to_complex_query"; public static final String HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD = "history_input_table_statistics_matching_threshold"; @@ -1492,6 +1493,11 @@ public SystemSessionProperties( "Enable history based optimization only for complex queries, i.e. queries with join and aggregation", true, false), + booleanProperty( + HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES, + "Estimate the size of the plan node output with variable statistics for HBO", + featuresConfig.isHistoryBasedOptimizerEstimateSizeUsingVariables(), + false), new PropertyMetadata<>( HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, "When the size difference between current table and history table exceed this threshold, do not match history statistics", @@ -2941,6 +2947,11 @@ public static boolean restrictHistoryBasedOptimizationToComplexQuery(Session ses return session.getSystemProperty(RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY, Boolean.class); } + public static boolean estimateSizeUsingVariablesForHBO(Session session) + { + return session.getSystemProperty(HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES, Boolean.class); + } + public static double getHistoryInputTableStatisticsMatchingThreshold(Session session) { return session.getSystemProperty(HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, Double.class); diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java b/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java index 65cd001801876..a903dd9dce459 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java @@ -36,6 +36,7 @@ import java.util.function.Supplier; import static com.facebook.presto.SystemSessionProperties.enableVerboseHistoryBasedOptimizerRuntimeStats; +import static com.facebook.presto.SystemSessionProperties.estimateSizeUsingVariablesForHBO; import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit; import static com.facebook.presto.SystemSessionProperties.getHistoryInputTableStatisticsMatchingThreshold; import static com.facebook.presto.SystemSessionProperties.isVerboseRuntimeStatsEnabled; @@ -203,7 +204,7 @@ private PlanNodeStatsEstimate getStatistics(PlanNode planNode, Session session, if ((toConfidenceLevel(predictedPlanStatistics.getConfidence()).getConfidenceOrdinal() >= delegateStats.confidenceLevel().getConfidenceOrdinal())) { return delegateStats.combineStats( predictedPlanStatistics, - new HistoryBasedSourceInfo(entry.getKey().getHash(), inputTableStatistics, Optional.ofNullable(historicalPlanStatisticsEntry.get().getHistoricalPlanStatisticsEntryInfo()))); + new HistoryBasedSourceInfo(entry.getKey().getHash(), inputTableStatistics, Optional.ofNullable(historicalPlanStatisticsEntry.get().getHistoricalPlanStatisticsEntryInfo()), estimateSizeUsingVariablesForHBO(session))); } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java b/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java index 2c02e64c85cde..e8c7432ff0079 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java @@ -59,6 +59,7 @@ import java.util.Set; import java.util.function.Supplier; +import static com.facebook.presto.SystemSessionProperties.estimateSizeUsingVariablesForHBO; import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit; import static com.facebook.presto.SystemSessionProperties.trackHistoryBasedPlanStatisticsEnabled; import static com.facebook.presto.SystemSessionProperties.trackHistoryStatsFromFailedQuery; @@ -242,7 +243,7 @@ else if (trackStatsForFailedQueries) { PlanStatisticsWithSourceInfo planStatsWithSourceInfo = new PlanStatisticsWithSourceInfo( planNode.getId(), newPlanNodeStats, - new HistoryBasedSourceInfo(Optional.of(hash), Optional.of(inputTableStatistics), Optional.of(historicalPlanStatisticsEntryInfo))); + new HistoryBasedSourceInfo(Optional.of(hash), Optional.of(inputTableStatistics), Optional.of(historicalPlanStatisticsEntryInfo), estimateSizeUsingVariablesForHBO(session))); planStatisticsMap.put(planNodeWithHash, planStatsWithSourceInfo); if (isAggregation(planNode, AggregationNode.Step.FINAL) && ((AggregationNode) planNode).getAggregationId().isPresent() && trackPartialAggregationHistory(session)) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index a92a1ab8c7faf..0b6a050598a8e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -106,6 +106,7 @@ public class FeaturesConfig private String historyBasedOptimizerPlanCanonicalizationStrategies = "IGNORE_SAFE_CONSTANTS"; private boolean logPlansUsedInHistoryBasedOptimizer; private boolean enforceTimeoutForHBOQueryRegistration; + private boolean historyBasedOptimizerEstimateSizeUsingVariables; private boolean redistributeWrites; private boolean scaleWriters = true; private DataSize writerMinSize = new DataSize(32, MEGABYTE); @@ -889,6 +890,18 @@ public FeaturesConfig setEnforceTimeoutForHBOQueryRegistration(boolean enforceTi return this; } + public boolean isHistoryBasedOptimizerEstimateSizeUsingVariables() + { + return historyBasedOptimizerEstimateSizeUsingVariables; + } + + @Config("optimizer.history-based-optimizer-estimate-size-using-variables") + public FeaturesConfig setHistoryBasedOptimizerEstimateSizeUsingVariables(boolean historyBasedOptimizerEstimateSizeUsingVariables) + { + this.historyBasedOptimizerEstimateSizeUsingVariables = historyBasedOptimizerEstimateSizeUsingVariables; + return this; + } + public AggregationPartitioningMergingStrategy getAggregationPartitioningMergingStrategy() { return aggregationPartitioningMergingStrategy; diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index db27a9c9a0a07..1c276d39f1d06 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -87,6 +87,7 @@ public void testDefaults() .setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS") .setLogPlansUsedInHistoryBasedOptimizer(false) .setEnforceTimeoutForHBOQueryRegistration(false) + .setHistoryBasedOptimizerEstimateSizeUsingVariables(false) .setRedistributeWrites(false) .setScaleWriters(true) .setWriterMinSize(new DataSize(32, MEGABYTE)) @@ -310,6 +311,7 @@ public void testExplicitPropertyMappings() .put("optimizer.history-based-optimizer-plan-canonicalization-strategies", "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS") .put("optimizer.log-plans-used-in-history-based-optimizer", "true") .put("optimizer.enforce-timeout-for-hbo-query-registration", "true") + .put("optimizer.history-based-optimizer-estimate-size-using-variables", "true") .put("optimizer.history-based-optimizer-timeout", "1s") .put("redistribute-writes", "true") .put("scale-writers", "false") @@ -517,6 +519,7 @@ public void testExplicitPropertyMappings() .setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS") .setLogPlansUsedInHistoryBasedOptimizer(true) .setEnforceTimeoutForHBOQueryRegistration(true) + .setHistoryBasedOptimizerEstimateSizeUsingVariables(true) .setRedistributeWrites(true) .setScaleWriters(false) .setWriterMinSize(new DataSize(42, GIGABYTE)) diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java index 90ff6957e45ee..521e005f7ce78 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java @@ -29,12 +29,14 @@ public class HistoryBasedSourceInfo private final Optional hash; private final Optional> inputTableStatistics; private final Optional historicalPlanStatisticsEntryInfo; + private final boolean estimateSizeUsingVariables; - public HistoryBasedSourceInfo(Optional hash, Optional> inputTableStatistics, Optional historicalPlanStatisticsEntryInfo) + public HistoryBasedSourceInfo(Optional hash, Optional> inputTableStatistics, Optional historicalPlanStatisticsEntryInfo, boolean estimateSizeUsingVariables) { this.hash = requireNonNull(hash, "hash is null"); this.inputTableStatistics = requireNonNull(inputTableStatistics, "inputTableStatistics is null"); this.historicalPlanStatisticsEntryInfo = requireNonNull(historicalPlanStatisticsEntryInfo, "historicalPlanStatisticsEntryInfo is null"); + this.estimateSizeUsingVariables = estimateSizeUsingVariables; } public Optional getHash() @@ -82,4 +84,10 @@ public String getSourceInfoName() { return "HBO"; } + + @Override + public boolean estimateSizeUsingVariables() + { + return estimateSizeUsingVariables; + } } diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java index 1504c660badb9..a69a7e417b412 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java @@ -51,6 +51,7 @@ import static com.facebook.presto.SystemSessionProperties.CONFIDENCE_BASED_BROADCAST_ENABLED; import static com.facebook.presto.SystemSessionProperties.ENFORCE_HISTORY_BASED_OPTIMIZER_REGISTRATION_TIMEOUT; +import static com.facebook.presto.SystemSessionProperties.HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES; import static com.facebook.presto.SystemSessionProperties.HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT; import static com.facebook.presto.SystemSessionProperties.HISTORY_CANONICAL_PLAN_NODE_LIMIT; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; @@ -131,6 +132,28 @@ public void testHistoryBasedStatsCalculator() anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(3).withOutputSize(54))); } + @Test + public void testHistoryBasedStatsCalculatorUsingVariables() + { + Session useVariables = Session.builder(createSession()) + .setSystemProperty(HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES, "true") + .build(); + // CBO Statistics + assertPlan( + "SELECT * FROM nation where substr(name, 1, 1) = 'A'", + anyTree(node(FilterNode.class, any()).withOutputRowCount(Double.NaN))); + + // HBO Statistics + executeAndTrackHistory("SELECT * FROM nation where substr(name, 1, 1) = 'A'"); + assertPlan( + "SELECT * FROM nation where substr(name, 1, 1) = 'A'", + anyTree(node(FilterNode.class, any()).withOutputRowCount(2).withOutputSize(199))); + assertPlan( + useVariables, + "SELECT * FROM nation where substr(name, 1, 1) = 'A'", + anyTree(node(FilterNode.class, any()).withOutputRowCount(2).withOutputSize(256))); + } + @Test public void testHistoryBasedStatsCalculatorEnforceTimeOut() {