Skip to content

Use non partitioned dynamic filter limits for FTE #17831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arhimondr merged 3 commits into from
Jun 15, 2023
Merged

Use non partitioned dynamic filter limits for FTE #17831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
21aaab1
Refactor dynamic filtering configuration
arhimondr Jun 8, 2023
3873ada
Drop unused configuration parameters
arhimondr Jun 8, 2023
4dea674
Use non partitioned dynamic filter limits in FTE
arhimondr Jun 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 68 additions & 48 deletions core/trino-main/src/main/java/io/trino/execution/DynamicFilterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,32 @@ public class DynamicFilterConfig
private boolean enableCoordinatorDynamicFiltersDistribution = true;
private boolean enableLargeDynamicFilters;

Comment thread
arhimondr marked this conversation as resolved.
Outdated
private int smallBroadcastMaxDistinctValuesPerDriver = 1_000;
private DataSize smallBroadcastMaxSizePerDriver = DataSize.of(100, KILOBYTE);
private int smallBroadcastRangeRowLimitPerDriver = 2_000;
private DataSize smallBroadcastMaxSizePerOperator = DataSize.of(1, MEGABYTE);
/*
* dynamic-filtering.small.* and dynamic-filtering.large.* limits are applied when
* collected over a not pre-partitioned source (when join distribution type is
* REPLICATED or when FTE is enabled).
*
* dynamic-filtering.small-partitioned.* and dynamic-filtering.large-partitioned.*
* limits are applied when collected over a pre-partitioned source (when join
* distribution type is PARTITIONED and FTE is disabled).
*
* When FTE is enabled dynamic filters are always collected over non partitioned data,
* hence the dynamic-filtering.small.* and dynamic-filtering.large.* limits applied.
*/
private int smallMaxDistinctValuesPerDriver = 1_000;
private DataSize smallMaxSizePerDriver = DataSize.of(100, KILOBYTE);
private int smallRangeRowLimitPerDriver = 2_000;
private DataSize smallMaxSizePerOperator = DataSize.of(1, MEGABYTE);
private int smallPartitionedMaxDistinctValuesPerDriver = 100;
private DataSize smallPartitionedMaxSizePerDriver = DataSize.of(50, KILOBYTE);
private int smallPartitionedRangeRowLimitPerDriver = 500;
private DataSize smallPartitionedMaxSizePerOperator = DataSize.of(500, KILOBYTE);
private DataSize smallMaxSizePerFilter = DataSize.of(5, MEGABYTE);

private int largeBroadcastMaxDistinctValuesPerDriver = 10_000;
private DataSize largeBroadcastMaxSizePerDriver = DataSize.of(2, MEGABYTE);
private int largeBroadcastRangeRowLimitPerDriver = 20_000;
private DataSize largeBroadcastMaxSizePerOperator = DataSize.of(5, MEGABYTE);
private int largeMaxDistinctValuesPerDriver = 10_000;
private DataSize largeMaxSizePerDriver = DataSize.of(2, MEGABYTE);
private int largeRangeRowLimitPerDriver = 20_000;
private DataSize largeMaxSizePerOperator = DataSize.of(5, MEGABYTE);
private int largePartitionedMaxDistinctValuesPerDriver = 1_000;
private DataSize largePartitionedMaxSizePerDriver = DataSize.of(200, KILOBYTE);
private int largePartitionedRangeRowLimitPerDriver = 2_000;
Expand Down Expand Up @@ -100,54 +112,58 @@ public DynamicFilterConfig setEnableLargeDynamicFilters(boolean enableLargeDynam
}

@Min(0)
public int getSmallBroadcastMaxDistinctValuesPerDriver()
public int getSmallMaxDistinctValuesPerDriver()
{
return smallBroadcastMaxDistinctValuesPerDriver;
return smallMaxDistinctValuesPerDriver;
}

@Config("dynamic-filtering.small-broadcast.max-distinct-values-per-driver")
public DynamicFilterConfig setSmallBroadcastMaxDistinctValuesPerDriver(int smallBroadcastMaxDistinctValuesPerDriver)
@LegacyConfig("dynamic-filtering.small-broadcast.max-distinct-values-per-driver")
@Config("dynamic-filtering.small.max-distinct-values-per-driver")
public DynamicFilterConfig setSmallMaxDistinctValuesPerDriver(int smallMaxDistinctValuesPerDriver)
{
this.smallBroadcastMaxDistinctValuesPerDriver = smallBroadcastMaxDistinctValuesPerDriver;
this.smallMaxDistinctValuesPerDriver = smallMaxDistinctValuesPerDriver;
return this;
}

@MaxDataSize("1MB")
public DataSize getSmallBroadcastMaxSizePerDriver()
public DataSize getSmallMaxSizePerDriver()
{
return smallBroadcastMaxSizePerDriver;
return smallMaxSizePerDriver;
}

@Config("dynamic-filtering.small-broadcast.max-size-per-driver")
public DynamicFilterConfig setSmallBroadcastMaxSizePerDriver(DataSize smallBroadcastMaxSizePerDriver)
@LegacyConfig("dynamic-filtering.small-broadcast.max-size-per-driver")
@Config("dynamic-filtering.small.max-size-per-driver")
public DynamicFilterConfig setSmallMaxSizePerDriver(DataSize smallMaxSizePerDriver)
{
this.smallBroadcastMaxSizePerDriver = smallBroadcastMaxSizePerDriver;
this.smallMaxSizePerDriver = smallMaxSizePerDriver;
return this;
}

@Min(0)
public int getSmallBroadcastRangeRowLimitPerDriver()
public int getSmallRangeRowLimitPerDriver()
{
return smallBroadcastRangeRowLimitPerDriver;
return smallRangeRowLimitPerDriver;
}

@Config("dynamic-filtering.small-broadcast.range-row-limit-per-driver")
public DynamicFilterConfig setSmallBroadcastRangeRowLimitPerDriver(int smallBroadcastRangeRowLimitPerDriver)
@LegacyConfig("dynamic-filtering.small-broadcast.range-row-limit-per-driver")
@Config("dynamic-filtering.small.range-row-limit-per-driver")
public DynamicFilterConfig setSmallRangeRowLimitPerDriver(int smallRangeRowLimitPerDriver)
{
this.smallBroadcastRangeRowLimitPerDriver = smallBroadcastRangeRowLimitPerDriver;
this.smallRangeRowLimitPerDriver = smallRangeRowLimitPerDriver;
return this;
}

@MaxDataSize("10MB")
public DataSize getSmallBroadcastMaxSizePerOperator()
public DataSize getSmallMaxSizePerOperator()
{
return smallBroadcastMaxSizePerOperator;
return smallMaxSizePerOperator;
}

@Config("dynamic-filtering.small-broadcast.max-size-per-operator")
public DynamicFilterConfig setSmallBroadcastMaxSizePerOperator(DataSize smallBroadcastMaxSizePerOperator)
@LegacyConfig("dynamic-filtering.small-broadcast.max-size-per-operator")
@Config("dynamic-filtering.small.max-size-per-operator")
public DynamicFilterConfig setSmallMaxSizePerOperator(DataSize smallMaxSizePerOperator)
{
this.smallBroadcastMaxSizePerOperator = smallBroadcastMaxSizePerOperator;
this.smallMaxSizePerOperator = smallMaxSizePerOperator;
return this;
}

Expand Down Expand Up @@ -218,53 +234,57 @@ public DynamicFilterConfig setSmallMaxSizePerFilter(DataSize smallMaxSizePerFilt
}

@Min(0)
public int getLargeBroadcastMaxDistinctValuesPerDriver()
public int getLargeMaxDistinctValuesPerDriver()
{
return largeBroadcastMaxDistinctValuesPerDriver;
return largeMaxDistinctValuesPerDriver;
}

@Config("dynamic-filtering.large-broadcast.max-distinct-values-per-driver")
public DynamicFilterConfig setLargeBroadcastMaxDistinctValuesPerDriver(int largeBroadcastMaxDistinctValuesPerDriver)
@LegacyConfig("dynamic-filtering.large-broadcast.max-distinct-values-per-driver")
@Config("dynamic-filtering.large.max-distinct-values-per-driver")
public DynamicFilterConfig setLargeMaxDistinctValuesPerDriver(int largeMaxDistinctValuesPerDriver)
{
this.largeBroadcastMaxDistinctValuesPerDriver = largeBroadcastMaxDistinctValuesPerDriver;
this.largeMaxDistinctValuesPerDriver = largeMaxDistinctValuesPerDriver;
return this;
}

public DataSize getLargeBroadcastMaxSizePerDriver()
public DataSize getLargeMaxSizePerDriver()
{
return largeBroadcastMaxSizePerDriver;
return largeMaxSizePerDriver;
}

@Config("dynamic-filtering.large-broadcast.max-size-per-driver")
public DynamicFilterConfig setLargeBroadcastMaxSizePerDriver(DataSize largeBroadcastMaxSizePerDriver)
@LegacyConfig("dynamic-filtering.large-broadcast.max-size-per-driver")
@Config("dynamic-filtering.large.max-size-per-driver")
public DynamicFilterConfig setLargeMaxSizePerDriver(DataSize largeMaxSizePerDriver)
{
this.largeBroadcastMaxSizePerDriver = largeBroadcastMaxSizePerDriver;
this.largeMaxSizePerDriver = largeMaxSizePerDriver;
return this;
}

@Min(0)
public int getLargeBroadcastRangeRowLimitPerDriver()
public int getLargeRangeRowLimitPerDriver()
{
return largeBroadcastRangeRowLimitPerDriver;
return largeRangeRowLimitPerDriver;
}

@Config("dynamic-filtering.large-broadcast.range-row-limit-per-driver")
public DynamicFilterConfig setLargeBroadcastRangeRowLimitPerDriver(int largeBroadcastRangeRowLimitPerDriver)
@LegacyConfig("dynamic-filtering.large-broadcast.range-row-limit-per-driver")
@Config("dynamic-filtering.large.range-row-limit-per-driver")
public DynamicFilterConfig setLargeRangeRowLimitPerDriver(int largeRangeRowLimitPerDriver)
{
this.largeBroadcastRangeRowLimitPerDriver = largeBroadcastRangeRowLimitPerDriver;
this.largeRangeRowLimitPerDriver = largeRangeRowLimitPerDriver;
return this;
}

@MaxDataSize("100MB")
public DataSize getLargeBroadcastMaxSizePerOperator()
public DataSize getLargeMaxSizePerOperator()
{
return largeBroadcastMaxSizePerOperator;
return largeMaxSizePerOperator;
}

@Config("dynamic-filtering.large-broadcast.max-size-per-operator")
public DynamicFilterConfig setLargeBroadcastMaxSizePerOperator(DataSize largeBroadcastMaxSizePerOperator)
@LegacyConfig("dynamic-filtering.large-broadcast.max-size-per-operator")
@Config("dynamic-filtering.large.max-size-per-operator")
public DynamicFilterConfig setLargeMaxSizePerOperator(DataSize largeMaxSizePerOperator)
{
this.largeBroadcastMaxSizePerOperator = largeBroadcastMaxSizePerOperator;
this.largeMaxSizePerOperator = largeMaxSizePerOperator;
return this;
}

Expand Down
Loading