Add an option to have HBO return output size with individual variable size

presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java

@@ -261,6 +261,7 @@ public final class SystemSessionProperties
     public static final String TRACK_HISTORY_STATS_FROM_FAILED_QUERIES = "track_history_stats_from_failed_queries";
     public static final String USE_PERFECTLY_CONSISTENT_HISTORIES = "use_perfectly_consistent_histories";
     public static final String HISTORY_CANONICAL_PLAN_NODE_LIMIT = "history_canonical_plan_node_limit";
+    public static final String HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES = "history_based_optimizer_estimate_size_using_variables";
     public static final String HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT = "history_based_optimizer_timeout_limit";
     public static final String RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY = "restrict_history_based_optimization_to_complex_query";
     public static final String HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD = "history_input_table_statistics_matching_threshold";
@@ -1492,6 +1493,11 @@ public SystemSessionProperties(
                         "Enable history based optimization only for complex queries, i.e. queries with join and aggregation",
                         true,
                         false),
+                booleanProperty(
+                        HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES,
+                        "Estimate the size of the plan node output with variable statistics for HBO",
+                        featuresConfig.isHistoryBasedOptimizerEstimateSizeUsingVariables(),
+                        false),
                 new PropertyMetadata<>(
                         HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD,
                         "When the size difference between current table and history table exceed this threshold, do not match history statistics",
@@ -2941,6 +2947,11 @@ public static boolean restrictHistoryBasedOptimizationToComplexQuery(Session ses
         return session.getSystemProperty(RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY, Boolean.class);
     }
 
+    public static boolean estimateSizeUsingVariablesForHBO(Session session)
+    {
+        return session.getSystemProperty(HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES, Boolean.class);
+    }
+
     public static double getHistoryInputTableStatisticsMatchingThreshold(Session session)
     {
         return session.getSystemProperty(HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, Double.class);

presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java

@@ -36,6 +36,7 @@
 import java.util.function.Supplier;
 
 import static com.facebook.presto.SystemSessionProperties.enableVerboseHistoryBasedOptimizerRuntimeStats;
+import static com.facebook.presto.SystemSessionProperties.estimateSizeUsingVariablesForHBO;
 import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit;
 import static com.facebook.presto.SystemSessionProperties.getHistoryInputTableStatisticsMatchingThreshold;
 import static com.facebook.presto.SystemSessionProperties.isVerboseRuntimeStatsEnabled;
@@ -203,7 +204,7 @@ private PlanNodeStatsEstimate getStatistics(PlanNode planNode, Session session,
                             if ((toConfidenceLevel(predictedPlanStatistics.getConfidence()).getConfidenceOrdinal() >= delegateStats.confidenceLevel().getConfidenceOrdinal())) {
                                 return delegateStats.combineStats(
                                         predictedPlanStatistics,
-                                        new HistoryBasedSourceInfo(entry.getKey().getHash(), inputTableStatistics, Optional.ofNullable(historicalPlanStatisticsEntry.get().getHistoricalPlanStatisticsEntryInfo())));
+                                        new HistoryBasedSourceInfo(entry.getKey().getHash(), inputTableStatistics, Optional.ofNullable(historicalPlanStatisticsEntry.get().getHistoricalPlanStatisticsEntryInfo()), estimateSizeUsingVariablesForHBO(session)));
                             }
                         }
                     }

presto-main-base/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java

@@ -59,6 +59,7 @@
 import java.util.Set;
 import java.util.function.Supplier;
 
+import static com.facebook.presto.SystemSessionProperties.estimateSizeUsingVariablesForHBO;
 import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit;
 import static com.facebook.presto.SystemSessionProperties.trackHistoryBasedPlanStatisticsEnabled;
 import static com.facebook.presto.SystemSessionProperties.trackHistoryStatsFromFailedQuery;
@@ -242,7 +243,7 @@ else if (trackStatsForFailedQueries) {
                         PlanStatisticsWithSourceInfo planStatsWithSourceInfo = new PlanStatisticsWithSourceInfo(
                                 planNode.getId(),
                                 newPlanNodeStats,
-                                new HistoryBasedSourceInfo(Optional.of(hash), Optional.of(inputTableStatistics), Optional.of(historicalPlanStatisticsEntryInfo)));
+                                new HistoryBasedSourceInfo(Optional.of(hash), Optional.of(inputTableStatistics), Optional.of(historicalPlanStatisticsEntryInfo), estimateSizeUsingVariablesForHBO(session)));
                         planStatisticsMap.put(planNodeWithHash, planStatsWithSourceInfo);
 
                         if (isAggregation(planNode, AggregationNode.Step.FINAL) && ((AggregationNode) planNode).getAggregationId().isPresent() && trackPartialAggregationHistory(session)) {

presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java

@@ -106,6 +106,7 @@ public class FeaturesConfig
     private String historyBasedOptimizerPlanCanonicalizationStrategies = "IGNORE_SAFE_CONSTANTS";
     private boolean logPlansUsedInHistoryBasedOptimizer;
     private boolean enforceTimeoutForHBOQueryRegistration;
+    private boolean historyBasedOptimizerEstimateSizeUsingVariables;
     private boolean redistributeWrites;
     private boolean scaleWriters = true;
     private DataSize writerMinSize = new DataSize(32, MEGABYTE);
@@ -889,6 +890,18 @@ public FeaturesConfig setEnforceTimeoutForHBOQueryRegistration(boolean enforceTi
         return this;
     }
 
+    public boolean isHistoryBasedOptimizerEstimateSizeUsingVariables()
+    {
+        return historyBasedOptimizerEstimateSizeUsingVariables;
+    }
+
+    @Config("optimizer.history-based-optimizer-estimate-size-using-variables")
+    public FeaturesConfig setHistoryBasedOptimizerEstimateSizeUsingVariables(boolean historyBasedOptimizerEstimateSizeUsingVariables)
+    {
+        this.historyBasedOptimizerEstimateSizeUsingVariables = historyBasedOptimizerEstimateSizeUsingVariables;
+        return this;
+    }
+
     public AggregationPartitioningMergingStrategy getAggregationPartitioningMergingStrategy()
     {
         return aggregationPartitioningMergingStrategy;

presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java

@@ -87,6 +87,7 @@ public void testDefaults()
                 .setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS")
                 .setLogPlansUsedInHistoryBasedOptimizer(false)
                 .setEnforceTimeoutForHBOQueryRegistration(false)
+                .setHistoryBasedOptimizerEstimateSizeUsingVariables(false)
                 .setRedistributeWrites(false)
                 .setScaleWriters(true)
                 .setWriterMinSize(new DataSize(32, MEGABYTE))
@@ -310,6 +311,7 @@ public void testExplicitPropertyMappings()
                 .put("optimizer.history-based-optimizer-plan-canonicalization-strategies", "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS")
                 .put("optimizer.log-plans-used-in-history-based-optimizer", "true")
                 .put("optimizer.enforce-timeout-for-hbo-query-registration", "true")
+                .put("optimizer.history-based-optimizer-estimate-size-using-variables", "true")
                 .put("optimizer.history-based-optimizer-timeout", "1s")
                 .put("redistribute-writes", "true")
                 .put("scale-writers", "false")
@@ -517,6 +519,7 @@ public void testExplicitPropertyMappings()
                 .setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS")
                 .setLogPlansUsedInHistoryBasedOptimizer(true)
                 .setEnforceTimeoutForHBOQueryRegistration(true)
+                .setHistoryBasedOptimizerEstimateSizeUsingVariables(true)
                 .setRedistributeWrites(true)
                 .setScaleWriters(false)
                 .setWriterMinSize(new DataSize(42, GIGABYTE))

presto-spi/src/main/java/com/facebook/presto/spi/statistics/HistoryBasedSourceInfo.java

@@ -29,12 +29,14 @@ public class HistoryBasedSourceInfo
     private final Optional<String> hash;
     private final Optional<List<PlanStatistics>> inputTableStatistics;
     private final Optional<HistoricalPlanStatisticsEntryInfo> historicalPlanStatisticsEntryInfo;
+    private final boolean estimateSizeUsingVariables;
 
-    public HistoryBasedSourceInfo(Optional<String> hash, Optional<List<PlanStatistics>> inputTableStatistics, Optional<HistoricalPlanStatisticsEntryInfo> historicalPlanStatisticsEntryInfo)
+    public HistoryBasedSourceInfo(Optional<String> hash, Optional<List<PlanStatistics>> inputTableStatistics, Optional<HistoricalPlanStatisticsEntryInfo> historicalPlanStatisticsEntryInfo, boolean estimateSizeUsingVariables)
     {
         this.hash = requireNonNull(hash, "hash is null");
         this.inputTableStatistics = requireNonNull(inputTableStatistics, "inputTableStatistics is null");
         this.historicalPlanStatisticsEntryInfo = requireNonNull(historicalPlanStatisticsEntryInfo, "historicalPlanStatisticsEntryInfo is null");
+        this.estimateSizeUsingVariables = estimateSizeUsingVariables;
     }
 
     public Optional<String> getHash()
@@ -82,4 +84,10 @@ public String getSourceInfoName()
     {
         return "HBO";
     }
+
+    @Override
+    public boolean estimateSizeUsingVariables()
+    {
+        return estimateSizeUsingVariables;
+    }
 }

presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java

@@ -51,6 +51,7 @@
 
 import static com.facebook.presto.SystemSessionProperties.CONFIDENCE_BASED_BROADCAST_ENABLED;
 import static com.facebook.presto.SystemSessionProperties.ENFORCE_HISTORY_BASED_OPTIMIZER_REGISTRATION_TIMEOUT;
+import static com.facebook.presto.SystemSessionProperties.HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES;
 import static com.facebook.presto.SystemSessionProperties.HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT;
 import static com.facebook.presto.SystemSessionProperties.HISTORY_CANONICAL_PLAN_NODE_LIMIT;
 import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
@@ -131,6 +132,28 @@ public void testHistoryBasedStatsCalculator()
                 anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(3).withOutputSize(54)));
     }
 
+    @Test
+    public void testHistoryBasedStatsCalculatorUsingVariables()
+    {
+        Session useVariables = Session.builder(createSession())
+                .setSystemProperty(HISTORY_BASED_OPTIMIZER_ESTIMATE_SIZE_USING_VARIABLES, "true")
+                .build();
+        // CBO Statistics
+        assertPlan(
+                "SELECT * FROM nation where substr(name, 1, 1) = 'A'",
+                anyTree(node(FilterNode.class, any()).withOutputRowCount(Double.NaN)));
+
+        // HBO Statistics
+        executeAndTrackHistory("SELECT * FROM nation where substr(name, 1, 1) = 'A'");
+        assertPlan(
+                "SELECT * FROM nation where substr(name, 1, 1) = 'A'",
+                anyTree(node(FilterNode.class, any()).withOutputRowCount(2).withOutputSize(199)));
+        assertPlan(
+                useVariables,
+                "SELECT * FROM nation where substr(name, 1, 1) = 'A'",
+                anyTree(node(FilterNode.class, any()).withOutputRowCount(2).withOutputSize(256)));
+    }
+
     @Test
     public void testHistoryBasedStatsCalculatorEnforceTimeOut()
     {