diff --git a/docs/src/main/sphinx/connector/bigquery.rst b/docs/src/main/sphinx/connector/bigquery.rst index 5301b44f0a82..b8e399aeb725 100644 --- a/docs/src/main/sphinx/connector/bigquery.rst +++ b/docs/src/main/sphinx/connector/bigquery.rst @@ -122,6 +122,7 @@ Property Description ``bigquery.views-enabled`` Enables the connector to read from views and not only tables. ``false`` Please read `this section <#reading-from-views>`_ before enabling this feature. +``bigquery.view-expire-duration`` Expire duration for the materialized view. ``24h`` ``bigquery.view-materialization-project`` The project where the materialized view is going to be created The view's project ``bigquery.view-materialization-dataset`` The dataset where the materialized view is going to be created The view's dataset ``bigquery.views-cache-ttl`` Duration for which the materialization of a view will be ``15m`` diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml index 0a4df62006e6..9e80b5504170 100644 --- a/plugin/trino-bigquery/pom.xml +++ b/plugin/trino-bigquery/pom.xml @@ -205,6 +205,11 @@ grpc-api + + javax.annotation + javax.annotation-api + + javax.inject javax.inject diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java index 22c636c273d2..143f3853e8f8 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConfig.java @@ -20,11 +20,13 @@ import io.airlift.units.Duration; import io.airlift.units.MinDuration; +import javax.annotation.PostConstruct; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import java.util.Optional; +import static com.google.common.base.Preconditions.checkState; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; @@ -38,6 +40,7 @@ public class BigQueryConfig private Optional parentProjectId = Optional.empty(); private Optional parallelism = Optional.empty(); private boolean viewsEnabled; + private Duration viewExpireDuration = new Duration(24, HOURS); private Optional viewMaterializationProject = Optional.empty(); private Optional viewMaterializationDataset = Optional.empty(); private int maxReadRowsRetries = DEFAULT_MAX_READ_ROWS_RETRIES; @@ -98,9 +101,17 @@ public BigQueryConfig setViewsEnabled(boolean viewsEnabled) return this; } - public Duration getViewExpiration() + @NotNull + public Duration getViewExpireDuration() + { + return viewExpireDuration; + } + + @Config("bigquery.view-expire-duration") + public BigQueryConfig setViewExpireDuration(Duration viewExpireDuration) { - return new Duration(24, HOURS); + this.viewExpireDuration = viewExpireDuration; + return this; } public Optional getViewMaterializationProject() @@ -186,4 +197,10 @@ public BigQueryConfig setServiceCacheTtl(Duration serviceCacheTtl) this.serviceCacheTtl = serviceCacheTtl; return this; } + + @PostConstruct + public void validate() + { + checkState(viewExpireDuration.toMillis() > viewsCacheTtl.toMillis(), "View expiration duration must be longer than view cache TTL"); + } } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java index 43aedecbafab..a61c1a279bb7 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java @@ -74,7 +74,7 @@ public BigQuerySplitManager( this.bigQueryReadClientFactory = requireNonNull(bigQueryReadClientFactory, "bigQueryReadClientFactory cannot be null"); this.parallelism = config.getParallelism(); this.viewEnabled = config.isViewsEnabled(); - this.viewExpiration = config.getViewExpiration(); + this.viewExpiration = config.getViewExpireDuration(); this.nodeManager = requireNonNull(nodeManager, "nodeManager cannot be null"); } diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java index 0ae541bad84e..31c100619455 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java @@ -71,6 +71,7 @@ public static DistributedQueryRunner createQueryRunner(Map extra connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties)); connectorProperties.putIfAbsent("bigquery.views-enabled", "true"); + connectorProperties.putIfAbsent("bigquery.view-expire-duration", "30m"); queryRunner.installPlugin(new BigQueryPlugin()); queryRunner.createCatalog( diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java index 9aae513ccf5e..9e7d3116d035 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConfig.java @@ -23,7 +23,9 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestBigQueryConfig { @@ -34,6 +36,7 @@ public void testDefaults() .setProjectId(null) .setParentProjectId(null) .setParallelism(null) + .setViewExpireDuration(new Duration(24, HOURS)) .setViewMaterializationProject(null) .setViewMaterializationDataset(null) .setMaxReadRowsRetries(3) @@ -51,6 +54,7 @@ public void testExplicitPropertyMappingsWithCredentialsKey() .put("bigquery.parent-project-id", "ppid") .put("bigquery.parallelism", "20") .put("bigquery.views-enabled", "true") + .put("bigquery.view-expire-duration", "30m") .put("bigquery.view-materialization-project", "vmproject") .put("bigquery.view-materialization-dataset", "vmdataset") .put("bigquery.max-read-rows-retries", "10") @@ -64,6 +68,7 @@ public void testExplicitPropertyMappingsWithCredentialsKey() .setParentProjectId("ppid") .setParallelism(20) .setViewsEnabled(true) + .setViewExpireDuration(new Duration(30, MINUTES)) .setViewMaterializationProject("vmproject") .setViewMaterializationDataset("vmdataset") .setMaxReadRowsRetries(10) @@ -73,4 +78,15 @@ public void testExplicitPropertyMappingsWithCredentialsKey() assertFullMapping(properties, expected); } + + @Test + public void testInvalidViewSetting() + { + assertThatThrownBy(() -> new BigQueryConfig() + .setViewExpireDuration(new Duration(5, MINUTES)) + .setViewsCacheTtl(new Duration(10, MINUTES)) + .validate()) + .isInstanceOf(IllegalStateException.class) + .hasMessage("View expiration duration must be longer than view cache TTL"); + } }