Skip to content

Remove fine-grained spill toggles #10591

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
findepi merged 2 commits into from
Jan 14, 2022
Merged

Remove fine-grained spill toggles #10591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f20c361
Remove fine-grained spill toggles
findepi Jan 13, 2022
3a14cf9
empty
findepi Jan 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 4 additions & 28 deletions core/trino-main/src/main/java/io/trino/FeaturesConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@
"optimizer.iterative-rule-based-column-pruning",
"optimizer.processing-optimization",
"resource-group-manager",
"spill-order-by",
"experimental.spill-order-by",
"spill-window-operator",
"experimental.spill-window-operator",
})
public class FeaturesConfig
{
Expand Down Expand Up @@ -105,8 +109,6 @@ public class FeaturesConfig
private int re2JDfaRetries = 5;
private RegexLibrary regexLibrary = JONI;
private boolean spillEnabled;
private boolean spillOrderBy = true;
private boolean spillWindowOperator = true;
private DataSize aggregationOperatorUnspillMemoryLimit = DataSize.of(4, DataSize.Unit.MEGABYTE);
private List<Path> spillerSpillPaths = ImmutableList.of();
private int spillerThreads = 4;
Expand Down Expand Up @@ -573,32 +575,6 @@ public FeaturesConfig setSpillEnabled(boolean spillEnabled)
return this;
}

public boolean isSpillOrderBy()
{
return spillOrderBy;
}

@Config("spill-order-by")
@LegacyConfig("experimental.spill-order-by")
public FeaturesConfig setSpillOrderBy(boolean spillOrderBy)
{
this.spillOrderBy = spillOrderBy;
return this;
}

public boolean isSpillWindowOperator()
{
return spillWindowOperator;
}

@Config("spill-window-operator")
@LegacyConfig("experimental.spill-window-operator")
public FeaturesConfig setSpillWindowOperator(boolean spillWindowOperator)
{
this.spillWindowOperator = spillWindowOperator;
return this;
}

public Duration getIterativeOptimizerTimeout()
{
return iterativeOptimizerTimeout;
Expand Down
22 changes: 0 additions & 22 deletions core/trino-main/src/main/java/io/trino/SystemSessionProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,6 @@ public final class SystemSessionProperties
public static final String OPTIMIZE_METADATA_QUERIES = "optimize_metadata_queries";
public static final String QUERY_PRIORITY = "query_priority";
public static final String SPILL_ENABLED = "spill_enabled";
public static final String SPILL_ORDER_BY = "spill_order_by";
public static final String SPILL_WINDOW_OPERATOR = "spill_window_operator";
public static final String AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT = "aggregation_operator_unspill_memory_limit";
public static final String OPTIMIZE_DISTINCT_AGGREGATIONS = "optimize_mixed_distinct_aggregations";
public static final String ITERATIVE_OPTIMIZER_TIMEOUT = "iterative_optimizer_timeout";
Expand Down Expand Up @@ -394,16 +392,6 @@ public SystemSessionProperties(
"Enable spilling",
featuresConfig.isSpillEnabled(),
false),
booleanProperty(
SPILL_ORDER_BY,
"Spill in OrderBy if spill_enabled is also set",
featuresConfig.isSpillOrderBy(),
false),
booleanProperty(
SPILL_WINDOW_OPERATOR,
"Spill in WindowOperator if spill_enabled is also set",
featuresConfig.isSpillWindowOperator(),
false),
dataSizeProperty(
AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT,
"How much memory should be allocated per aggregation operator in unspilling process",
Expand Down Expand Up @@ -926,16 +914,6 @@ public static boolean isSpillEnabled(Session session)
return session.getSystemProperty(SPILL_ENABLED, Boolean.class);
}

public static boolean isSpillOrderBy(Session session)
{
return session.getSystemProperty(SPILL_ORDER_BY, Boolean.class);
}

public static boolean isSpillWindowOperator(Session session)
{
return session.getSystemProperty(SPILL_WINDOW_OPERATOR, Boolean.class);
}

public static DataSize getAggregationOperatorUnspillMemoryLimit(Session session)
{
DataSize memoryLimitForMerge = session.getSystemProperty(AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT, DataSize.class);
Expand Down
8 changes: 3 additions & 5 deletions core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,8 +285,6 @@
import static io.trino.SystemSessionProperties.isExchangeCompressionEnabled;
import static io.trino.SystemSessionProperties.isLateMaterializationEnabled;
import static io.trino.SystemSessionProperties.isSpillEnabled;
import static io.trino.SystemSessionProperties.isSpillOrderBy;
import static io.trino.SystemSessionProperties.isSpillWindowOperator;
import static io.trino.operator.DistinctLimitOperator.DistinctLimitOperatorFactory;
import static io.trino.operator.HashArraySizeSupplier.incrementalLoadFactorHashArraySizeSupplier;
import static io.trino.operator.PipelineExecutionStrategy.GROUPED_EXECUTION;
Expand Down Expand Up @@ -1146,7 +1144,7 @@ public PhysicalOperation visitWindow(WindowNode node, LocalExecutionPlanContext
node.getPreSortedOrderPrefix(),
10_000,
pagesIndexFactory,
isSpillEnabled(session) && isSpillWindowOperator(session),
isSpillEnabled(session),
spillerFactory,
orderingCompiler,
ImmutableList.of(),
Expand Down Expand Up @@ -1402,7 +1400,7 @@ public PhysicalOperation visitPatternRecognition(PatternRecognitionNode node, Lo
node.getPreSortedOrderPrefix(),
10_000,
pagesIndexFactory,
isSpillEnabled(session) && isSpillWindowOperator(session),
isSpillEnabled(session),
spillerFactory,
orderingCompiler,
node.getMeasures().values().stream()
Expand Down Expand Up @@ -1654,7 +1652,7 @@ public PhysicalOperation visitSort(SortNode node, LocalExecutionPlanContext cont
outputChannels.add(i);
}

boolean spillEnabled = isSpillEnabled(session) && isSpillOrderBy(session);
boolean spillEnabled = isSpillEnabled(session);

OperatorFactory operator = new OrderByOperatorFactory(
context.getNextOperatorId(),
Expand Down
6 changes: 0 additions & 6 deletions core/trino-main/src/test/java/io/trino/sql/analyzer/TestFeaturesConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,6 @@ public void testDefaults()
.setRe2JDfaStatesLimit(Integer.MAX_VALUE)
.setRe2JDfaRetries(5)
.setSpillEnabled(false)
.setSpillOrderBy(true)
.setSpillWindowOperator(true)
.setAggregationOperatorUnspillMemoryLimit(DataSize.valueOf("4MB"))
.setSpillerSpillPaths("")
.setSpillerThreads(4)
Expand Down Expand Up @@ -160,8 +158,6 @@ public void testExplicitPropertyMappings()
.put("re2j.dfa-states-limit", "42")
.put("re2j.dfa-retries", "42")
.put("spill-enabled", "true")
.put("spill-order-by", "false")
.put("spill-window-operator", "false")
.put("aggregation-operator-unspill-memory-limit", "100MB")
.put("spiller-spill-path", "/tmp/custom/spill/path1,/tmp/custom/spill/path2")
.put("spiller-threads", "42")
Expand Down Expand Up @@ -237,8 +233,6 @@ public void testExplicitPropertyMappings()
.setRe2JDfaStatesLimit(42)
.setRe2JDfaRetries(42)
.setSpillEnabled(true)
.setSpillOrderBy(false)
.setSpillWindowOperator(false)
.setAggregationOperatorUnspillMemoryLimit(DataSize.valueOf("100MB"))
.setSpillerSpillPaths("/tmp/custom/spill/path1,/tmp/custom/spill/path2")
.setSpillerThreads(42)
Expand Down
22 changes: 0 additions & 22 deletions docs/src/main/sphinx/admin/properties-spilling.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,6 @@ reduce memory usage required for other join types.

This config property can be overridden by the ``spill_enabled`` session property.

``spill-order-by``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

* **Type:** :ref:`prop-type-boolean`
* **Default value:** ``true``

Try spilling memory to disk to avoid exceeding memory limits for the query when running sorting operators.
This property must be used in conjunction with the ``spill-enabled`` property.

This config property can be overridden by the ``spill_order_by`` session property.

``spill-window-operator``
^^^^^^^^^^^^^^^^^^^^^^^^^

* **Type:** :ref:`prop-type-boolean`
* **Default value:** ``true``

Try spilling memory to disk to avoid exceeding memory limits for the query when running window operators.
This property must be used in conjunction with the ``spill-enabled`` property.

This config property can be overridden by the ``spill_window_operator`` session property.

``spiller-spill-path``
^^^^^^^^^^^^^^^^^^^^^^

Expand Down
1 change: 0 additions & 1 deletion testing/trino-tests/src/test/java/io/trino/tests/TestDistributedSpilledQueries.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ public static DistributedQueryRunner createSpillingQueryRunner()
.setSchema(TINY_SCHEMA_NAME)
.setSystemProperty(SystemSessionProperties.TASK_CONCURRENCY, "2")
.setSystemProperty(SystemSessionProperties.SPILL_ENABLED, "true")
.setSystemProperty(SystemSessionProperties.SPILL_ORDER_BY, "true")
.setSystemProperty(SystemSessionProperties.AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT, "128kB")
.build();

Expand Down