diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java index 833ef43a8544..15e50d8c3173 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeModule.java @@ -14,6 +14,7 @@ package io.trino.plugin.deltalake; import com.google.inject.Binder; +import com.google.inject.Key; import com.google.inject.Provider; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -55,6 +56,7 @@ import io.trino.plugin.hive.metastore.HiveMetastore; import io.trino.plugin.hive.metastore.MetastoreConfig; import io.trino.plugin.hive.metastore.SemiTransactionalHiveMetastore; +import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; import io.trino.plugin.hive.procedure.OptimizeTableProcedure; @@ -92,6 +94,7 @@ public void setup(Binder binder) configBinder(binder).bindConfig(DeltaLakeConfig.class); configBinder(binder).bindConfig(HiveConfig.class); binder.bind(MetastoreConfig.class).toInstance(new MetastoreConfig()); // currently not configurable + binder.bind(Key.get(boolean.class, TranslateHiveViews.class)).toInstance(false); configBinder(binder).bindConfig(ParquetReaderConfig.class); configBinder(binder).bindConfig(ParquetWriterConfig.class); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java index 4db428bfd971..28aac0e3e8ba 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java @@ -63,6 +63,7 @@ public class HiveConfig { private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + public static final String HIVE_VIEWS_ENABLED = "hive.hive-views.enabled"; private boolean singleStatementWritesOnly; @@ -771,7 +772,7 @@ public boolean isTranslateHiveViews() } @LegacyConfig({"hive.views-execution.enabled", "hive.translate-hive-views"}) - @Config("hive.hive-views.enabled") + @Config(HIVE_VIEWS_ENABLED) @ConfigDescription("Experimental: Allow translation of Hive views into Trino views") public HiveConfig setTranslateHiveViews(boolean translateHiveViews) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java index 6f8868ef99b4..98a8aa0ea346 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java @@ -23,6 +23,7 @@ import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.fs.CachingDirectoryLister; import io.trino.plugin.hive.metastore.MetastoreConfig; +import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.hive.orc.OrcFileWriterFactory; import io.trino.plugin.hive.orc.OrcPageSourceFactory; import io.trino.plugin.hive.orc.OrcReaderConfig; @@ -139,4 +140,12 @@ public ScheduledExecutorService createHiveTransactionHeartbeatExecutor(CatalogNa hiveConfig.getHiveTransactionHeartbeatThreads(), daemonThreadsNamed("hive-heartbeat-" + catalogName + "-%s")); } + + @TranslateHiveViews + @Singleton + @Provides + public boolean translateHiveViews(HiveConfig hiveConfig) + { + return hiveConfig.isTranslateHiveViews(); + } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java index 9a825451e371..be9730c9ff63 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastore.java @@ -25,7 +25,6 @@ import io.trino.plugin.hive.HdfsEnvironment; import io.trino.plugin.hive.HdfsEnvironment.HdfsContext; import io.trino.plugin.hive.HiveBasicStatistics; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HivePartition; import io.trino.plugin.hive.HiveType; import io.trino.plugin.hive.PartitionNotFoundException; @@ -197,16 +196,16 @@ public class ThriftHiveMetastore @Inject public ThriftHiveMetastore( MetastoreLocator metastoreLocator, - HiveConfig hiveConfig, MetastoreConfig metastoreConfig, + @TranslateHiveViews boolean translateHiveViews, ThriftMetastoreConfig thriftConfig, ThriftMetastoreAuthenticationConfig authenticationConfig, HdfsEnvironment hdfsEnvironment) { this( metastoreLocator, - hiveConfig, metastoreConfig, + translateHiveViews, thriftConfig, hdfsEnvironment, authenticationConfig.getAuthenticationType() != ThriftMetastoreAuthenticationType.NONE); @@ -214,8 +213,8 @@ public ThriftHiveMetastore( public ThriftHiveMetastore( MetastoreLocator metastoreLocator, - HiveConfig hiveConfig, MetastoreConfig metastoreConfig, + boolean translateHiveViews, ThriftMetastoreConfig thriftConfig, HdfsEnvironment hdfsEnvironment, boolean authenticationEnabled) @@ -230,8 +229,7 @@ public ThriftHiveMetastore( this.maxRetries = thriftConfig.getMaxRetries(); this.impersonationEnabled = thriftConfig.isImpersonationEnabled(); this.deleteFilesOnDrop = thriftConfig.isDeleteFilesOnDrop(); - requireNonNull(hiveConfig, "hiveConfig is null"); - this.translateHiveViews = hiveConfig.isTranslateHiveViews(); + this.translateHiveViews = translateHiveViews; requireNonNull(metastoreConfig, "metastoreConfig is null"); checkArgument(!metastoreConfig.isHideDeltaLakeTables(), "Hiding Delta Lake tables is not supported"); // TODO this.maxWaitForLock = thriftConfig.getMaxWaitForTransactionLock(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TranslateHiveViews.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TranslateHiveViews.java new file mode 100644 index 000000000000..74fe473854a7 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/TranslateHiveViews.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.metastore.thrift; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface TranslateHiveViews +{ +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index 2216dd0c24ba..9f0bb4260311 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -791,8 +791,8 @@ protected final void setup(String host, int port, String databaseName, String ti HiveMetastore metastore = cachingHiveMetastore( new BridgingHiveMetastore(new ThriftHiveMetastore( metastoreLocator, - hiveConfig, new MetastoreConfig(), + new HiveConfig().isTranslateHiveViews(), new ThriftMetastoreConfig(), hdfsEnvironment, false), diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java index baaa382e5922..8a50e1eacdec 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHiveFileSystem.java @@ -190,8 +190,8 @@ protected void setup(String host, int port, String databaseName, boolean s3Selec new BridgingHiveMetastore( new ThriftHiveMetastore( metastoreLocator, - new HiveConfig(), metastoreConfig, + new HiveConfig().isTranslateHiveViews(), new ThriftMetastoreConfig(), hdfsEnvironment, false), diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java index e2ed7b7940aa..89f14f329510 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java @@ -80,8 +80,8 @@ protected QueryRunner createQueryRunner() new TestingMetastoreLocator( Optional.empty(), this.dockerizedS3DataLake.getHiveHadoop().getHiveMetastoreEndpoint()), - new HiveConfig(), new MetastoreConfig(), + new HiveConfig().isTranslateHiveViews(), new ThriftMetastoreConfig(), new HdfsEnvironment(new HiveHdfsConfiguration( new HdfsConfigurationInitializer( diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java index bf7fbf27ef65..245bd50920b0 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java @@ -154,7 +154,7 @@ private ThriftHiveMetastore createThriftHiveMetastore() private static ThriftHiveMetastore createThriftHiveMetastore(ThriftMetastoreClient client) { MetastoreLocator metastoreLocator = new MockMetastoreLocator(client); - return new ThriftHiveMetastore(metastoreLocator, new HiveConfig(), new MetastoreConfig(), new ThriftMetastoreConfig(), HDFS_ENVIRONMENT, false); + return new ThriftHiveMetastore(metastoreLocator, new MetastoreConfig(), new HiveConfig().isTranslateHiveViews(), new ThriftMetastoreConfig(), HDFS_ENVIRONMENT, false); } @Test diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java index b658a6555347..184c22d1ba0d 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/s3/S3HiveQueryRunner.java @@ -156,8 +156,8 @@ public DistributedQueryRunner build() Optional.empty(), hiveMetastoreEndpoint, metastoreTimeout), - new HiveConfig(), new MetastoreConfig(), + new HiveConfig().isTranslateHiveViews(), new ThriftMetastoreConfig(), new HdfsEnvironment(new HiveHdfsConfiguration( new HdfsConfigurationInitializer( diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java index 843a0c1b117a..10efb526281d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java @@ -15,6 +15,8 @@ import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.LegacyConfig; +import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.plugin.hive.HiveCompressionCodec; @@ -24,6 +26,7 @@ import java.util.Optional; +import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.trino.plugin.hive.HiveCompressionCodec.ZSTD; import static io.trino.plugin.iceberg.CatalogType.HIVE_METASTORE; import static io.trino.plugin.iceberg.IcebergFileFormat.ORC; @@ -50,6 +53,11 @@ public class IcebergConfig private int formatVersion = FORMAT_VERSION_SUPPORT_MAX; private Duration expireSnapshotsMinRetention = new Duration(7, DAYS); private Duration removeOrphanFilesMinRetention = new Duration(7, DAYS); + private DataSize targetMaxFileSize = DataSize.of(1, GIGABYTE); + // This is meant to protect users who are misusing schema locations (by + // putting schemas in locations with extraneous files), so default to false + // to avoid deleting those files if Trino is unable to check. + private boolean deleteSchemaLocationsFallback; public CatalogType getCatalogType() { @@ -235,4 +243,32 @@ public IcebergConfig setRemoveOrphanFilesMinRetention(Duration removeOrphanFiles this.removeOrphanFilesMinRetention = removeOrphanFilesMinRetention; return this; } + + public DataSize getTargetMaxFileSize() + { + return targetMaxFileSize; + } + + @LegacyConfig("hive.target-max-file-size") + @Config("iceberg.target-max-file-size") + @ConfigDescription("Target maximum size of written files; the actual size may be larger") + public IcebergConfig setTargetMaxFileSize(DataSize targetMaxFileSize) + { + this.targetMaxFileSize = targetMaxFileSize; + return this; + } + + public boolean isDeleteSchemaLocationsFallback() + { + return this.deleteSchemaLocationsFallback; + } + + @LegacyConfig("hive.delete-schema-locations-fallback") + @Config("iceberg.delete-schema-locations-fallback") + @ConfigDescription("Whether schema locations should be deleted when Trino can't determine whether they contain external files.") + public IcebergConfig setDeleteSchemaLocationsFallback(boolean deleteSchemaLocationsFallback) + { + this.deleteSchemaLocationsFallback = deleteSchemaLocationsFallback; + return this; + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index 19763aaba540..69546d8ffc7a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -19,7 +19,6 @@ import com.google.inject.multibindings.Multibinder; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.hive.FileFormatDataSourceStats; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.metastore.MetastoreConfig; import io.trino.plugin.hive.orc.OrcReaderConfig; import io.trino.plugin.hive.orc.OrcWriterConfig; @@ -48,8 +47,6 @@ public class IcebergModule public void configure(Binder binder) { binder.bind(IcebergTransactionManager.class).in(Scopes.SINGLETON); - - configBinder(binder).bindConfig(HiveConfig.class); configBinder(binder).bindConfig(IcebergConfig.class); configBinder(binder).bindConfig(MetastoreConfig.class); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java index 09d94ff70bf7..39923d934a1f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java @@ -19,7 +19,6 @@ import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.hive.HiveCompressionCodec; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.orc.OrcReaderConfig; import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.hive.parquet.ParquetReaderConfig; @@ -85,8 +84,7 @@ public IcebergSessionProperties( OrcReaderConfig orcReaderConfig, OrcWriterConfig orcWriterConfig, ParquetReaderConfig parquetReaderConfig, - ParquetWriterConfig parquetWriterConfig, - HiveConfig hiveConfig) + ParquetWriterConfig parquetWriterConfig) { sessionProperties = ImmutableList.>builder() .add(enumProperty( @@ -222,7 +220,7 @@ public IcebergSessionProperties( .add(dataSizeProperty( TARGET_MAX_FILE_SIZE, "Target maximum size of written files; the actual size may be larger", - hiveConfig.getTargetMaxFileSize(), + icebergConfig.getTargetMaxFileSize(), false)) .add(stringProperty( HIVE_CATALOG_NAME, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java index 018d6e4ae6ed..ea23d0f103f1 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java @@ -14,10 +14,12 @@ package io.trino.plugin.iceberg.catalog.hms; import com.google.inject.Binder; +import com.google.inject.Key; import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.hive.metastore.DecoratedHiveMetastoreModule; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreModule; +import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.iceberg.catalog.IcebergCatalogModule.MetastoreValidator; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; @@ -32,6 +34,7 @@ protected void setup(Binder binder) binder.bind(IcebergTableOperationsProvider.class).to(HiveMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); binder.bind(MetastoreValidator.class).asEagerSingleton(); + binder.bind(Key.get(boolean.class, TranslateHiveViews.class)).toInstance(false); install(new DecoratedHiveMetastoreModule()); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java index 9310f0be102f..018618a618db 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java @@ -15,7 +15,6 @@ import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.HdfsEnvironment; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.iceberg.IcebergConfig; @@ -56,8 +55,7 @@ public TrinoHiveCatalogFactory( TypeManager typeManager, IcebergTableOperationsProvider tableOperationsProvider, NodeVersion nodeVersion, - IcebergSecurityConfig securityConfig, - HiveConfig hiveConfig) + IcebergSecurityConfig securityConfig) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.metastoreFactory = requireNonNull(metastoreFactory, "metastoreFactory is null"); @@ -69,8 +67,7 @@ public TrinoHiveCatalogFactory( this.isUniqueTableLocation = config.isUniqueTableLocation(); requireNonNull(securityConfig, "securityConfig is null"); this.isUsingSystemSecurity = securityConfig.getSecuritySystem() == SYSTEM; - requireNonNull(hiveConfig, "hiveConfig is null"); - this.deleteSchemaLocationsFallback = hiveConfig.isDeleteSchemaLocationsFallback(); + this.deleteSchemaLocationsFallback = config.isDeleteSchemaLocationsFallback(); } @Override diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java index b4e653be808d..0c4d14aa2ae8 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java @@ -14,6 +14,7 @@ package io.trino.plugin.iceberg; import com.google.common.collect.ImmutableMap; +import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.plugin.hive.HiveCompressionCodec; import org.testng.annotations.Test; @@ -23,6 +24,8 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.trino.plugin.hive.HiveCompressionCodec.ZSTD; import static io.trino.plugin.iceberg.CatalogType.GLUE; import static io.trino.plugin.iceberg.CatalogType.HIVE_METASTORE; @@ -50,7 +53,9 @@ public void testDefaults() .setHiveCatalogName(null) .setFormatVersion(2) .setExpireSnapshotsMinRetention(new Duration(7, DAYS)) - .setRemoveOrphanFilesMinRetention(new Duration(7, DAYS))); + .setRemoveOrphanFilesMinRetention(new Duration(7, DAYS)) + .setDeleteSchemaLocationsFallback(false) + .setTargetMaxFileSize(DataSize.of(1, GIGABYTE))); } @Test @@ -70,6 +75,8 @@ public void testExplicitPropertyMappings() .put("iceberg.format-version", "1") .put("iceberg.expire_snapshots.min-retention", "13h") .put("iceberg.remove_orphan_files.min-retention", "14h") + .put("iceberg.delete-schema-locations-fallback", "true") + .put("iceberg.target-max-file-size", "1MB") .buildOrThrow(); IcebergConfig expected = new IcebergConfig() @@ -85,7 +92,9 @@ public void testExplicitPropertyMappings() .setHiveCatalogName("hive") .setFormatVersion(1) .setExpireSnapshotsMinRetention(new Duration(13, HOURS)) - .setRemoveOrphanFilesMinRetention(new Duration(14, HOURS)); + .setRemoveOrphanFilesMinRetention(new Duration(14, HOURS)) + .setDeleteSchemaLocationsFallback(true) + .setTargetMaxFileSize(DataSize.of(1, MEGABYTE)); assertFullMapping(properties, expected); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergNodeLocalDynamicSplitPruning.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergNodeLocalDynamicSplitPruning.java index 0487f52fe8b9..e0332349248d 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergNodeLocalDynamicSplitPruning.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergNodeLocalDynamicSplitPruning.java @@ -27,7 +27,6 @@ import io.trino.orc.OrcWriterStats; import io.trino.orc.OutputStreamOrcDataSink; import io.trino.plugin.hive.FileFormatDataSourceStats; -import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HiveTransactionHandle; import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.Column; @@ -105,16 +104,15 @@ public void testDynamicSplitPruning() throws IOException { IcebergConfig icebergConfig = new IcebergConfig(); - HiveConfig hiveConfig = new HiveConfig(); HiveTransactionHandle transaction = new HiveTransactionHandle(false); try (TempFile tempFile = new TempFile()) { writeOrcContent(tempFile.file()); - try (ConnectorPageSource emptyPageSource = createTestingPageSource(transaction, icebergConfig, hiveConfig, tempFile.file(), getDynamicFilter(getTupleDomainForSplitPruning()))) { + try (ConnectorPageSource emptyPageSource = createTestingPageSource(transaction, icebergConfig, tempFile.file(), getDynamicFilter(getTupleDomainForSplitPruning()))) { assertNull(emptyPageSource.getNextPage()); } - try (ConnectorPageSource nonEmptyPageSource = createTestingPageSource(transaction, icebergConfig, hiveConfig, tempFile.file(), getDynamicFilter(getNonSelectiveTupleDomain()))) { + try (ConnectorPageSource nonEmptyPageSource = createTestingPageSource(transaction, icebergConfig, tempFile.file(), getDynamicFilter(getNonSelectiveTupleDomain()))) { Page page = nonEmptyPageSource.getNextPage(); assertNotNull(page); assertEquals(page.getBlock(0).getPositionCount(), 1); @@ -151,7 +149,7 @@ private static void writeOrcContent(File file) } } - private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle transaction, IcebergConfig icebergConfig, HiveConfig hiveConfig, File outputFile, DynamicFilter dynamicFilter) + private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle transaction, IcebergConfig icebergConfig, File outputFile, DynamicFilter dynamicFilter) { IcebergSplit split = new IcebergSplit( "file:///" + outputFile.getAbsolutePath(), @@ -199,7 +197,7 @@ private static ConnectorPageSource createTestingPageSource(HiveTransactionHandle return provider.createPageSource( transaction, - getSession(icebergConfig, hiveConfig), + getSession(icebergConfig), split, tableHandle.getConnectorHandle(), ImmutableList.of(KEY_ICEBERG_COLUMN_HANDLE, DATA_ICEBERG_COLUMN_HANDLE), @@ -222,10 +220,10 @@ private static TupleDomain getNonSelectiveTupleDomain() Domain.singleValue(INTEGER, (long) KEY_COLUMN_VALUE))); } - private static TestingConnectorSession getSession(IcebergConfig icebergConfig, HiveConfig hiveConfig) + private static TestingConnectorSession getSession(IcebergConfig icebergConfig) { return TestingConnectorSession.builder() - .setPropertyMetadata(new IcebergSessionProperties(icebergConfig, ORC_READER_CONFIG, ORC_WRITER_CONFIG, PARQUET_READER_CONFIG, PARQUET_WRITER_CONFIG, hiveConfig).getSessionProperties()) + .setPropertyMetadata(new IcebergSessionProperties(icebergConfig, ORC_READER_CONFIG, ORC_WRITER_CONFIG, PARQUET_READER_CONFIG, PARQUET_WRITER_CONFIG).getSessionProperties()) .build(); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPlugin.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPlugin.java index 22a2e46c164a..f2516b8c806c 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPlugin.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPlugin.java @@ -14,6 +14,7 @@ package io.trino.plugin.iceberg; import com.google.common.collect.ImmutableMap; +import io.airlift.bootstrap.ApplicationConfigurationException; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorFactory; import io.trino.testing.TestingConnectorContext; @@ -24,6 +25,7 @@ import java.util.Map; import static com.google.common.collect.Iterables.getOnlyElement; +import static io.trino.plugin.hive.HiveConfig.HIVE_VIEWS_ENABLED; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -192,6 +194,23 @@ public void testFileBasedAccessControl() .shutdown(); } + @Test + public void testIcebergPluginFailsWhenIncorrectPropertyProvided() + { + ConnectorFactory factory = getConnectorFactory(); + + assertThatThrownBy(() -> factory.create( + "test", + Map.of( + "iceberg.catalog.type", "HIVE_METASTORE", + HIVE_VIEWS_ENABLED, "true", + "hive.metastore.uri", "thrift://foo:1234"), + new TestingConnectorContext()) + .shutdown()) + .isInstanceOf(ApplicationConfigurationException.class) + .hasMessageContaining("Configuration property 'hive.hive-views.enabled' was not used"); + } + private static ConnectorFactory getConnectorFactory() { return getOnlyElement(new IcebergPlugin().getConnectorFactories());