diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java index f51856f559dc..1010fb065163 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java @@ -16,14 +16,17 @@ import com.google.common.annotations.VisibleForTesting; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.LegacyConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.plugin.hive.HiveCompressionCodec; +import org.joda.time.DateTimeZone; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import java.util.Optional; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -58,6 +61,9 @@ public class DeltaLakeConfig private boolean tableStatisticsEnabled = true; private boolean extendedStatisticsEnabled = true; private HiveCompressionCodec compressionCodec = HiveCompressionCodec.SNAPPY; + private long perTransactionMetastoreCacheMaximumSize = 1000; + private boolean deleteSchemaLocationsFallback; + private String parquetTimeZone = TimeZone.getDefault().getID(); public Duration getMetadataCacheTtl() { @@ -311,4 +317,52 @@ public DeltaLakeConfig setCompressionCodec(HiveCompressionCodec compressionCodec this.compressionCodec = compressionCodec; return this; } + + @Min(1) + public long getPerTransactionMetastoreCacheMaximumSize() + { + return perTransactionMetastoreCacheMaximumSize; + } + + @LegacyConfig("hive.per-transaction-metastore-cache-maximum-size") + @Config("delta.per-transaction-metastore-cache-maximum-size") + public DeltaLakeConfig setPerTransactionMetastoreCacheMaximumSize(long perTransactionMetastoreCacheMaximumSize) + { + this.perTransactionMetastoreCacheMaximumSize = perTransactionMetastoreCacheMaximumSize; + return this; + } + + public boolean isDeleteSchemaLocationsFallback() + { + return this.deleteSchemaLocationsFallback; + } + + @LegacyConfig("hive.per-transaction-metastore-cache-maximum-size") + @Config("delta.delete-schema-locations-fallback") + @ConfigDescription("Whether schema locations should be deleted when Trino can't determine whether they contain external files.") + public DeltaLakeConfig setDeleteSchemaLocationsFallback(boolean deleteSchemaLocationsFallback) + { + this.deleteSchemaLocationsFallback = deleteSchemaLocationsFallback; + return this; + } + + public DateTimeZone getParquetDateTimeZone() + { + return DateTimeZone.forID(parquetTimeZone); + } + + @NotNull + public String getParquetTimeZone() + { + return parquetTimeZone; + } + + @LegacyConfig("hive.parquet.time-zone") + @Config("delta.parquet.time-zone") + @ConfigDescription("Time zone for Parquet read and write") + public DeltaLakeConfig setParquetTimeZone(String parquetTimeZone) + { + this.parquetTimeZone = parquetTimeZone; + return this; + } } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index 3f2f2df86e4f..061b68b1990c 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -20,7 +20,6 @@ import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointWriterManager; import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; import io.trino.plugin.hive.HdfsEnvironment; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.spi.NodeManager; @@ -69,8 +68,7 @@ public DeltaLakeMetadataFactory( NodeManager nodeManager, CheckpointWriterManager checkpointWriterManager, DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider, - CachingExtendedStatisticsAccess statisticsAccess, - HiveConfig hiveConfig) + CachingExtendedStatisticsAccess statisticsAccess) { this.hiveMetastoreFactory = requireNonNull(hiveMetastoreFactory, "hiveMetastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); @@ -89,8 +87,8 @@ public DeltaLakeMetadataFactory( this.unsafeWritesEnabled = deltaLakeConfig.getUnsafeWritesEnabled(); this.checkpointWritingInterval = deltaLakeConfig.getDefaultCheckpointWritingInterval(); this.ignoreCheckpointWriteFailures = deltaLakeConfig.isIgnoreCheckpointWriteFailures(); - this.perTransactionMetastoreCacheMaximumSize = hiveConfig.getPerTransactionMetastoreCacheMaximumSize(); - this.deleteSchemaLocationsFallback = hiveConfig.isDeleteSchemaLocationsFallback(); + this.perTransactionMetastoreCacheMaximumSize = deltaLakeConfig.getPerTransactionMetastoreCacheMaximumSize(); + this.deleteSchemaLocationsFallback = deltaLakeConfig.isDeleteSchemaLocationsFallback(); } public DeltaLakeMetadata create(ConnectorIdentity identity) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java index 25f376aecde5..1d20aaecdaa5 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java @@ -43,7 +43,6 @@ import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogSynchronizerManager; import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; import io.trino.plugin.hive.FileFormatDataSourceStats; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HiveLocationService; import io.trino.plugin.hive.HiveTransactionHandle; import io.trino.plugin.hive.HiveTransactionManager; @@ -91,7 +90,6 @@ public void setup(Binder binder) Provider catalogName = binder.getProvider(CatalogName.class); configBinder(binder).bindConfig(DeltaLakeConfig.class); - configBinder(binder).bindConfig(HiveConfig.class); binder.bind(Key.get(boolean.class, TranslateHiveViews.class)).toInstance(false); configBinder(binder).bindConfig(ParquetReaderConfig.class); configBinder(binder).bindConfig(ParquetWriterConfig.class); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java index 490d16341f65..2696940ab5e2 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java @@ -21,7 +21,6 @@ import io.trino.plugin.hive.HdfsEnvironment; import io.trino.plugin.hive.HdfsEnvironment.HdfsContext; import io.trino.plugin.hive.HiveColumnHandle; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.ReaderPageSource; import io.trino.plugin.hive.parquet.ParquetPageSourceFactory; import io.trino.plugin.hive.parquet.ParquetReaderConfig; @@ -73,7 +72,6 @@ public DeltaLakePageSourceProvider( FileFormatDataSourceStats fileFormatDataSourceStats, ParquetReaderConfig parquetReaderConfig, DeltaLakeConfig deltaLakeConfig, - HiveConfig hiveConfig, ExecutorService executorService, TypeManager typeManager, JsonCodec updateResultJsonCodec) @@ -82,7 +80,7 @@ public DeltaLakePageSourceProvider( this.fileFormatDataSourceStats = requireNonNull(fileFormatDataSourceStats, "fileFormatDataSourceStats is null"); this.parquetReaderOptions = requireNonNull(parquetReaderConfig, "parquetReaderConfig is null").toParquetReaderOptions(); this.domainCompactionThreshold = requireNonNull(deltaLakeConfig, "deltaLakeConfig is null").getDomainCompactionThreshold(); - this.parquetDateTimeZone = requireNonNull(hiveConfig, "hiveConfig is null").getParquetDateTimeZone(); + this.parquetDateTimeZone = deltaLakeConfig.getParquetDateTimeZone(); this.executorService = requireNonNull(executorService, "executorService is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.updateResultJsonCodec = requireNonNull(updateResultJsonCodec, "deleteResultJsonCodec is null"); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java index 7680b4a3e903..cb9766d79599 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java @@ -20,11 +20,13 @@ import org.testng.annotations.Test; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.trino.plugin.hive.util.TestHiveUtil.nonDefaultTimeZone; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; @@ -54,7 +56,10 @@ public void testDefaults() .setDynamicFilteringWaitTimeout(new Duration(0, SECONDS)) .setTableStatisticsEnabled(true) .setExtendedStatisticsEnabled(true) - .setCompressionCodec(HiveCompressionCodec.SNAPPY)); + .setCompressionCodec(HiveCompressionCodec.SNAPPY) + .setDeleteSchemaLocationsFallback(false) + .setParquetTimeZone(TimeZone.getDefault().getID()) + .setPerTransactionMetastoreCacheMaximumSize(1000)); } @Test @@ -80,6 +85,9 @@ public void testExplicitPropertyMappings() .put("delta.table-statistics-enabled", "false") .put("delta.extended-statistics.enabled", "false") .put("delta.compression-codec", "GZIP") + .put("delta.per-transaction-metastore-cache-maximum-size", "500") + .put("delta.delete-schema-locations-fallback", "true") + .put("delta.parquet.time-zone", nonDefaultTimeZone().getID()) .buildOrThrow(); DeltaLakeConfig expected = new DeltaLakeConfig() @@ -101,7 +109,10 @@ public void testExplicitPropertyMappings() .setDynamicFilteringWaitTimeout(new Duration(30, MINUTES)) .setTableStatisticsEnabled(false) .setExtendedStatisticsEnabled(false) - .setCompressionCodec(HiveCompressionCodec.GZIP); + .setCompressionCodec(HiveCompressionCodec.GZIP) + .setDeleteSchemaLocationsFallback(true) + .setParquetTimeZone(nonDefaultTimeZone().getID()) + .setPerTransactionMetastoreCacheMaximumSize(500); assertFullMapping(properties, expected); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java index 04d4ae35e12c..804a6a737570 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java @@ -151,7 +151,7 @@ public HiveMetastore createMetastore(Optional identity) deltaLakeProperties.put("hive.metastore", "test"); // use test value so we do not get clash with default bindings) if (!enablePerTransactionHiveMetastoreCaching) { // almost disable the cache; 0 is not allowed as config property value - deltaLakeProperties.put("hive.per-transaction-metastore-cache-maximum-size", "1"); + deltaLakeProperties.put("delta.per-transaction-metastore-cache-maximum-size", "1"); } queryRunner.createCatalog(DELTA_CATALOG, "delta-lake", deltaLakeProperties.buildOrThrow()); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index 6ec52f71ace0..fce96c20b8d6 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -14,11 +14,13 @@ package io.trino.plugin.iceberg; import com.google.inject.Binder; +import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Scopes; import com.google.inject.multibindings.Multibinder; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.hive.FileFormatDataSourceStats; +import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.hive.orc.OrcReaderConfig; import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.hive.parquet.ParquetReaderConfig; @@ -46,6 +48,7 @@ public class IcebergModule public void configure(Binder binder) { binder.bind(IcebergTransactionManager.class).in(Scopes.SINGLETON); + binder.bind(Key.get(boolean.class, TranslateHiveViews.class)).toInstance(false); configBinder(binder).bindConfig(IcebergConfig.class); newSetBinder(binder, SessionPropertiesProvider.class).addBinding().to(IcebergSessionProperties.class).in(Scopes.SINGLETON);