diff --git a/core/trino-main/src/main/java/io/trino/execution/DynamicFilterConfig.java b/core/trino-main/src/main/java/io/trino/execution/DynamicFilterConfig.java index 8ac74b3c878a..6f1b2e13001c 100644 --- a/core/trino-main/src/main/java/io/trino/execution/DynamicFilterConfig.java +++ b/core/trino-main/src/main/java/io/trino/execution/DynamicFilterConfig.java @@ -41,7 +41,7 @@ public class DynamicFilterConfig { private boolean enableDynamicFiltering = true; - private boolean enableLargeDynamicFilters; + private boolean enableLargeDynamicFilters = true; private boolean enableDynamicRowFiltering = true; private double dynamicRowFilterSelectivityThreshold = 0.7; @@ -67,15 +67,15 @@ public class DynamicFilterConfig private DataSize smallPartitionedMaxSizePerOperator = DataSize.of(500, KILOBYTE); private DataSize smallMaxSizePerFilter = DataSize.of(5, MEGABYTE); - private int largeMaxDistinctValuesPerDriver = 10_000; - private DataSize largeMaxSizePerDriver = DataSize.of(2, MEGABYTE); - private int largeRangeRowLimitPerDriver = 20_000; + private int largeMaxDistinctValuesPerDriver = 50_000; + private DataSize largeMaxSizePerDriver = DataSize.of(4, MEGABYTE); + private int largeRangeRowLimitPerDriver = 100_000; private DataSize largeMaxSizePerOperator = DataSize.of(5, MEGABYTE); - private int largePartitionedMaxDistinctValuesPerDriver = 1_000; + private int largePartitionedMaxDistinctValuesPerDriver = 20_000; private DataSize largePartitionedMaxSizePerDriver = DataSize.of(200, KILOBYTE); - private int largePartitionedRangeRowLimitPerDriver = 2_000; - private DataSize largePartitionedMaxSizePerOperator = DataSize.of(2, MEGABYTE); - private DataSize largeMaxSizePerFilter = DataSize.of(5, MEGABYTE); + private int largePartitionedRangeRowLimitPerDriver = 30_000; + private DataSize largePartitionedMaxSizePerOperator = DataSize.of(5, MEGABYTE); + private DataSize largeMaxSizePerFilter = DataSize.of(10, MEGABYTE); public boolean isEnableDynamicFiltering() { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestDynamicFilterConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestDynamicFilterConfig.java index ac5950ffcf05..88afd3e12189 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestDynamicFilterConfig.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestDynamicFilterConfig.java @@ -32,7 +32,7 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(DynamicFilterConfig.class) .setEnableDynamicFiltering(true) - .setEnableLargeDynamicFilters(false) + .setEnableLargeDynamicFilters(true) .setEnableDynamicRowFiltering(true) .setDynamicRowFilterSelectivityThreshold(0.7) .setSmallMaxDistinctValuesPerDriver(1_000) @@ -44,15 +44,15 @@ public void testDefaults() .setSmallPartitionedRangeRowLimitPerDriver(500) .setSmallPartitionedMaxSizePerOperator(DataSize.of(500, KILOBYTE)) .setSmallMaxSizePerFilter(DataSize.of(5, MEGABYTE)) - .setLargeMaxDistinctValuesPerDriver(10_000) - .setLargeMaxSizePerDriver(DataSize.of(2, MEGABYTE)) - .setLargeRangeRowLimitPerDriver(20_000) + .setLargeMaxDistinctValuesPerDriver(50_000) + .setLargeMaxSizePerDriver(DataSize.of(4, MEGABYTE)) + .setLargeRangeRowLimitPerDriver(100_000) .setLargeMaxSizePerOperator(DataSize.of(5, MEGABYTE)) - .setLargePartitionedMaxDistinctValuesPerDriver(1_000) + .setLargePartitionedMaxDistinctValuesPerDriver(20_000) .setLargePartitionedMaxSizePerDriver(DataSize.of(200, KILOBYTE)) - .setLargePartitionedRangeRowLimitPerDriver(2_000) - .setLargePartitionedMaxSizePerOperator(DataSize.of(2, MEGABYTE)) - .setLargeMaxSizePerFilter(DataSize.of(5, MEGABYTE))); + .setLargePartitionedRangeRowLimitPerDriver(30_000) + .setLargePartitionedMaxSizePerOperator(DataSize.of(5, MEGABYTE)) + .setLargeMaxSizePerFilter(DataSize.of(10, MEGABYTE))); } @Test @@ -60,7 +60,7 @@ public void testExplicitPropertyMappings() { Map properties = ImmutableMap.builder() .put("enable-dynamic-filtering", "false") - .put("enable-large-dynamic-filters", "true") + .put("enable-large-dynamic-filters", "false") .put("enable-dynamic-row-filtering", "false") .put("dynamic-row-filtering.selectivity-threshold", "0.8") .put("dynamic-filtering.small.max-distinct-values-per-driver", "256") @@ -85,7 +85,7 @@ public void testExplicitPropertyMappings() DynamicFilterConfig expected = new DynamicFilterConfig() .setEnableDynamicFiltering(false) - .setEnableLargeDynamicFilters(true) + .setEnableLargeDynamicFilters(false) .setEnableDynamicRowFiltering(false) .setDynamicRowFilterSelectivityThreshold(0.8) .setSmallMaxDistinctValuesPerDriver(256) diff --git a/core/trino-main/src/test/java/io/trino/server/TestDynamicFilterService.java b/core/trino-main/src/test/java/io/trino/server/TestDynamicFilterService.java index 948cffbf12b7..0cf131ef186d 100644 --- a/core/trino-main/src/test/java/io/trino/server/TestDynamicFilterService.java +++ b/core/trino-main/src/test/java/io/trino/server/TestDynamicFilterService.java @@ -63,6 +63,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.units.DataSize.Unit.KILOBYTE; +import static io.trino.SystemSessionProperties.ENABLE_LARGE_DYNAMIC_FILTERS; import static io.trino.SystemSessionProperties.RETRY_POLICY; import static io.trino.metadata.TestMetadataManager.createTestMetadataManager; import static io.trino.server.DynamicFilterService.DynamicFilterDomainStats; @@ -91,7 +92,9 @@ public class TestDynamicFilterService { - private static final Session session = TestingSession.testSessionBuilder().build(); + private static final Session session = TestingSession.testSessionBuilder() + .setSystemProperty(ENABLE_LARGE_DYNAMIC_FILTERS, "false") + .build(); @Test public void testDynamicFilterSummaryCompletion() @@ -835,7 +838,8 @@ public void testSizeLimit() { DataSize sizeLimit = DataSize.of(1, KILOBYTE); DynamicFilterConfig config = new DynamicFilterConfig(); - config.setSmallMaxSizePerFilter(sizeLimit); + config.setEnableLargeDynamicFilters(false) + .setSmallMaxSizePerFilter(sizeLimit); DynamicFilterService dynamicFilterService = new DynamicFilterService( PLANNER_CONTEXT.getMetadata(), PLANNER_CONTEXT.getFunctionManager(), diff --git a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java index fa41580eace7..abc54b106411 100644 --- a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java +++ b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java @@ -58,6 +58,7 @@ protected QueryRunner createQueryRunner() return MemoryQueryRunner.builder() .addExtraProperties(ImmutableMap.builder() // Adjust DF limits to test edge cases + .put("enable-large-dynamic-filters", "false") .put("dynamic-filtering.small.max-distinct-values-per-driver", "100") .put("dynamic-filtering.small.range-row-limit-per-driver", "100") .put("dynamic-filtering.large.max-distinct-values-per-driver", "100") diff --git a/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/TestFaultTolerantExecutionDynamicFiltering.java b/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/TestFaultTolerantExecutionDynamicFiltering.java index 727989052282..db2d233fd3d1 100644 --- a/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/TestFaultTolerantExecutionDynamicFiltering.java +++ b/testing/trino-faulttolerant-tests/src/test/java/io/trino/faulttolerant/TestFaultTolerantExecutionDynamicFiltering.java @@ -55,6 +55,7 @@ protected QueryRunner createQueryRunner() return DistributedQueryRunner.builder(getDefaultSession()) .setExtraProperties(FaultTolerantExecutionConnectorTestHelper.getExtraProperties()) // keep limits lower to test edge cases + .addExtraProperty("enable-large-dynamic-filters", "false") .addExtraProperty("dynamic-filtering.small.max-distinct-values-per-driver", "10") .addExtraProperty("dynamic-filtering.small.range-row-limit-per-driver", "100") .setAdditionalSetup(runner -> { diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestCoordinatorDynamicFiltering.java b/testing/trino-tests/src/test/java/io/trino/execution/TestCoordinatorDynamicFiltering.java index d85e6430513d..d7b8157f08e5 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestCoordinatorDynamicFiltering.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/TestCoordinatorDynamicFiltering.java @@ -34,6 +34,7 @@ protected QueryRunner createQueryRunner() .setExtraProperties(ImmutableMap.of( "retry-policy", getRetryPolicy().name(), // keep limits lower to test edge cases + "enable-large-dynamic-filters", "false", "dynamic-filtering.small-partitioned.max-distinct-values-per-driver", "10", "dynamic-filtering.small.max-distinct-values-per-driver", "10")) .build();