Skip to content

Add configs for ttl of Alluxio cache #19905

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zacw7 merged 1 commit into from
Jun 22, 2023
Merged

Add configs for ttl of Alluxio cache #19905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions presto-cache/src/main/java/com/facebook/presto/cache/alluxio/AlluxioCacheConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static io.airlift.units.DataSize.Unit.GIGABYTE;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.SECONDS;

public class AlluxioCacheConfig
Expand All @@ -38,6 +39,9 @@ public class AlluxioCacheConfig
private EvictionPolicy evictionPolicy = EvictionPolicy.LRU;
private boolean shadowCacheEnabled;
private Duration shadowCacheWindow = new Duration(7, DAYS);
private boolean ttlEnabled;
private Duration ttlCheckInterval = new Duration(1, HOURS);
private Duration ttlThreshold = new Duration(14, DAYS);

public boolean isMetricsCollectionEnabled()
{
Expand Down Expand Up @@ -220,4 +224,43 @@ public AlluxioCacheConfig setShadowCacheWindow(Duration shadowCacheWindow)
this.shadowCacheWindow = shadowCacheWindow;
return this;
}

@Config("cache.alluxio.ttl-enabled")
@ConfigDescription("If the alluxio caching enables the TTL")
public AlluxioCacheConfig setTtlEnabled(boolean ttlEnabled)
{
this.ttlEnabled = ttlEnabled;
return this;
}

public boolean isTtlEnabled()
{
return ttlEnabled;
}

public Duration getTtlCheckInterval()
{
return ttlCheckInterval;
}

@Config("cache.alluxio.ttl-check-interval")
@ConfigDescription("TTL check interval for alluxio cache")
public AlluxioCacheConfig setTtlCheckInterval(Duration ttlCheckInterval)
{
this.ttlCheckInterval = ttlCheckInterval;
return this;
}

public Duration getTtlThreshold()
{
return ttlThreshold;
}

@Config("cache.alluxio.ttl-threshold")
@ConfigDescription("TTL threshold for alluxio cache")
public AlluxioCacheConfig setTtlThreshold(Duration ttlThreshold)
{
this.ttlThreshold = ttlThreshold;
return this;
}
}
7 changes: 7 additions & 0 deletions .../src/main/java/com/facebook/presto/cache/alluxio/AlluxioCachingConfigurationProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ public void updateConfiguration(Configuration configuration, HdfsContext context
}
configuration.set("alluxio.user.client.cache.shadow.enabled", String.valueOf(alluxioCacheConfig.isShadowCacheEnabled()));
configuration.set("alluxio.user.client.cache.shadow.window", String.valueOf(alluxioCacheConfig.getShadowCacheWindow().toMillis()));
configuration.set("alluxio.user.client.cache.ttl.enabled", String.valueOf(alluxioCacheConfig.isTtlEnabled()));
if (alluxioCacheConfig.isTtlEnabled()) {
configuration.set("alluxio.user.client.cache.ttl.check.interval.seconds",
String.valueOf(alluxioCacheConfig.getTtlCheckInterval().toMillis() / 1000));
configuration.set("alluxio.user.client.cache.ttl.threshold.seconds",
String.valueOf(alluxioCacheConfig.getTtlThreshold().toMillis() / 1000));
}
}
}

Expand Down
15 changes: 13 additions & 2 deletions presto-cache/src/test/java/com/facebook/presto/cache/alluxio/TestAlluxioCacheConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import static io.airlift.units.DataSize.Unit.GIGABYTE;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;

public class TestAlluxioCacheConfig
Expand All @@ -47,7 +49,10 @@ public void testDefaults()
.setTimeoutThreads(64)
.setCacheQuotaEnabled(false)
.setShadowCacheEnabled(false)
.setShadowCacheWindow(new Duration(7, DAYS)));
.setShadowCacheWindow(new Duration(7, DAYS))
.setTtlCheckInterval(new Duration(1, HOURS))
.setTtlEnabled(false)
.setTtlThreshold(new Duration(14, DAYS)));
}

@Test
Expand All @@ -68,6 +73,9 @@ public void testExplicitPropertyMappings()
.put("cache.alluxio.quota-enabled", "true")
.put("cache.alluxio.shadow-cache-enabled", "true")
.put("cache.alluxio.shadow-cache-window", "1d")
.put("cache.alluxio.ttl-check-interval", "60s")
.put("cache.alluxio.ttl-enabled", "true")
.put("cache.alluxio.ttl-threshold", "60m")
.build();

AlluxioCacheConfig expected = new AlluxioCacheConfig()
Expand All @@ -84,7 +92,10 @@ public void testExplicitPropertyMappings()
.setTimeoutThreads(512)
.setCacheQuotaEnabled(true)
.setShadowCacheEnabled(true)
.setShadowCacheWindow(new Duration(1, DAYS));
.setShadowCacheWindow(new Duration(1, DAYS))
.setTtlCheckInterval(new Duration(60, SECONDS))
.setTtlEnabled(true)
.setTtlThreshold(new Duration(60, MINUTES));

assertFullMapping(properties, expected);
}
Expand Down