Skip to content

Disable stats and cost calculators by default #11765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arhimondr merged 2 commits into from
Oct 23, 2018
Merged

Disable stats and cost calculators by default #11765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c829482
Disable cost calculator if stats calculator is disabled
arhimondr Oct 23, 2018
4b46a29
Disable stats and cost calculators by default
arhimondr Oct 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions presto-main/src/main/java/com/facebook/presto/cost/CachingCostProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Optional;

import static com.facebook.presto.SystemSessionProperties.isEnableStatsCalculator;
import static com.google.common.base.Verify.verify;
import static java.util.Objects.requireNonNull;

Expand Down Expand Up @@ -54,6 +55,10 @@ public CachingCostProvider(CostCalculator costCalculator, StatsProvider statsPro
@Override
public PlanNodeCostEstimate getCumulativeCost(PlanNode node)
{
if (!isEnableStatsCalculator(session)) {
Copy link
Copy Markdown
Contributor

@electrum electrum Oct 23, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change seems unrelated to making it disabled by default and should probably be a separate commit?

Copy link
Copy Markdown
Contributor

@rschlussel rschlussel Oct 23, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this basically a shortcut so you don't keep trying to calculate cost and coming up empty because all stats are unknown?

Copy link
Copy Markdown
Member Author

@arhimondr arhimondr Oct 23, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this basically a shortcut so you don't keep trying to calculate cost and coming up empty because all stats are unknown?

I'm really being paranoid here. Just want to make sure that this part of functionality is really killed for good.

Copy link
Copy Markdown
Member Author

@arhimondr arhimondr Oct 23, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I extracted it into a separate commit

return PlanNodeCostEstimate.unknown();
}

requireNonNull(node, "node is null");

if (node instanceof GroupReference) {
Expand Down
2 changes: 1 addition & 1 deletion presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public class FeaturesConfig
private int spillerThreads = 4;
private double spillMaxUsedSpaceThreshold = 0.9;
private boolean iterativeOptimizerEnabled = true;
private boolean enableStatsCalculator = true;
private boolean enableStatsCalculator;
private boolean pushAggregationThroughJoin = true;
private double memoryRevokingTarget = 0.5;
private double memoryRevokingThreshold = 0.9;
Expand Down
6 changes: 5 additions & 1 deletion presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import java.util.Optional;
import java.util.function.Function;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.cost.PlanNodeCostEstimate.cpuCost;
import static com.facebook.presto.metadata.FunctionKind.AGGREGATE;
import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager;
Expand Down Expand Up @@ -116,7 +117,10 @@ public void setUp()
costCalculatorWithEstimatedExchanges = new CostCalculatorWithEstimatedExchanges(costCalculatorUsingExchanges, () -> NUMBER_OF_NODES);
planFragmenter = new PlanFragmenter(new QueryManagerConfig());

session = testSessionBuilder().setCatalog("tpch").build();
session = testSessionBuilder()
.setCatalog("tpch")
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.build();

CatalogManager catalogManager = new CatalogManager();
catalogManager.registerCatalog(createBogusTestingCatalog("tpch"));
Expand Down
2 changes: 2 additions & 0 deletions presto-main/src/test/java/com/facebook/presto/cost/TestStatsCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.ImmutableMap;
import org.testng.annotations.Test;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree;
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
Expand All @@ -37,6 +38,7 @@ public TestStatsCalculator()
this.queryRunner = new LocalQueryRunner(testSessionBuilder()
.setCatalog("local")
.setSchema("tiny")
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.setSystemProperty("task_concurrency", "1") // these tests don't handle exchanges from local parallel
.build());

Expand Down
6 changes: 3 additions & 3 deletions presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public void testDefaults()
.setLegacyLogFunction(false)
.setIterativeOptimizerEnabled(true)
.setIterativeOptimizerTimeout(new Duration(3, MINUTES))
.setEnableStatsCalculator(true)
.setEnableStatsCalculator(false)
.setExchangeCompressionEnabled(false)
.setLegacyTimestamp(true)
.setLegacyRowFieldOrdinalAccess(false)
Expand Down Expand Up @@ -117,7 +117,7 @@ public void testExplicitPropertyMappings()
.put("network-cost-weight", "0.2")
.put("experimental.iterative-optimizer-enabled", "false")
.put("experimental.iterative-optimizer-timeout", "10s")
.put("experimental.enable-stats-calculator", "false")
.put("experimental.enable-stats-calculator", "true")
.put("deprecated.legacy-array-agg", "true")
.put("deprecated.legacy-log-function", "true")
.put("deprecated.group-by-uses-equal", "true")
Expand Down Expand Up @@ -178,7 +178,7 @@ public void testExplicitPropertyMappings()
.setNetworkCostWeight(0.2)
.setIterativeOptimizerEnabled(false)
.setIterativeOptimizerTimeout(new Duration(10, SECONDS))
.setEnableStatsCalculator(false)
.setEnableStatsCalculator(true)
.setDistributedIndexJoinsEnabled(true)
.setJoinDistributionType(BROADCAST)
.setJoinMaxBroadcastTableSize(new DataSize(42, GIGABYTE))
Expand Down
6 changes: 5 additions & 1 deletion ...ava/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import java.util.Optional;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
import static com.facebook.presto.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE;
import static com.facebook.presto.spi.type.BigintType.BIGINT;
Expand All @@ -58,7 +59,10 @@ public class TestDetermineJoinDistributionType
@BeforeClass
public void setUp()
{
tester = new RuleTester(ImmutableList.of(), ImmutableMap.of(), Optional.of(4));
tester = new RuleTester(
ImmutableList.of(),
ImmutableMap.of(ENABLE_STATS_CALCULATOR, "true"),
Optional.of(4));
}

@AfterClass(alwaysRun = true)
Expand Down
4 changes: 3 additions & 1 deletion ...o-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.util.List;
import java.util.Optional;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
import static com.facebook.presto.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE;
import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
Expand Down Expand Up @@ -66,7 +67,8 @@ public void setUp()
ImmutableList.of(),
ImmutableMap.of(
JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name(),
JOIN_REORDERING_STRATEGY, JoinReorderingStrategy.AUTOMATIC.name()),
JOIN_REORDERING_STRATEGY, JoinReorderingStrategy.AUTOMATIC.name(),
ENABLE_STATS_CALCULATOR, "true"),
Optional.of(4));
}

Expand Down
2 changes: 2 additions & 0 deletions ...in/src/test/java/com/facebook/presto/sql/planner/optimizations/TestTpchCostBasedPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
Expand Down Expand Up @@ -53,6 +54,7 @@ public TestTpchCostBasedPlan()
.setCatalog(catalog)
.setSchema("sf3000.0")
.setSystemProperty("task_concurrency", "1") // these tests don't handle exchanges from local parallel
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.setSystemProperty(JOIN_REORDERING_STRATEGY, JoinReorderingStrategy.AUTOMATIC.name())
.setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name());

Expand Down
5 changes: 4 additions & 1 deletion presto-tests/src/test/java/com/facebook/presto/tests/TestTpchDistributedStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.SystemSessionProperties.PREFER_PARTITIAL_AGGREGATION;
import static com.facebook.presto.tests.statistics.MetricComparisonStrategies.absoluteError;
import static com.facebook.presto.tests.statistics.MetricComparisonStrategies.defaultTolerance;
Expand All @@ -41,7 +42,9 @@ public void setup()
{
DistributedQueryRunner runner = TpchQueryRunnerBuilder.builder()
// We are not able to calculate stats for PARTIAL aggregations
.amendSession(builder -> builder.setSystemProperty(PREFER_PARTITIAL_AGGREGATION, "false"))
.amendSession(builder -> builder
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.setSystemProperty(PREFER_PARTITIAL_AGGREGATION, "false"))
.buildWithoutCatalogs();
runner.createCatalog(
"tpch",
Expand Down
2 changes: 2 additions & 0 deletions presto-tests/src/test/java/com/facebook/presto/tests/TestTpchLocalStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.SystemSessionProperties.PREFER_PARTITIAL_AGGREGATION;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
import static com.facebook.presto.tests.statistics.MetricComparisonStrategies.absoluteError;
Expand All @@ -48,6 +49,7 @@ public void setUp()
.setCatalog("tpch")
.setSchema(TINY_SCHEMA_NAME)
// We are not able to calculate stats for PARTIAL aggregations
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.setSystemProperty(PREFER_PARTITIAL_AGGREGATION, "false")
.build();

Expand Down
2 changes: 2 additions & 0 deletions ...s/src/test/java/com/facebook/presto/sql/planner/optimizations/TestTpcdsCostBasedPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
Expand Down Expand Up @@ -51,6 +52,7 @@ public TestTpcdsCostBasedPlan()
.setCatalog(catalog)
.setSchema("sf3000.0")
.setSystemProperty("task_concurrency", "1") // these tests don't handle exchanges from local parallel
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.setSystemProperty(JOIN_REORDERING_STRATEGY, JoinReorderingStrategy.AUTOMATIC.name())
.setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name());

Expand Down
2 changes: 2 additions & 0 deletions presto-tpcds/src/test/java/com/facebook/presto/tpcds/statistics/TestTpcdsLocalStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import static com.facebook.presto.SystemSessionProperties.ENABLE_STATS_CALCULATOR;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
import static com.facebook.presto.tests.statistics.MetricComparisonStrategies.absoluteError;
import static com.facebook.presto.tests.statistics.MetricComparisonStrategies.defaultTolerance;
Expand All @@ -40,6 +41,7 @@ public void setUp()
Session defaultSession = testSessionBuilder()
.setCatalog("tpcds")
.setSchema("sf1")
.setSystemProperty(ENABLE_STATS_CALCULATOR, "true")
.build();

LocalQueryRunner queryRunner = new LocalQueryRunner(defaultSession);
Expand Down