diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ForFileIO.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ForFileIO.java new file mode 100644 index 000000000000..cee0c8dc0328 --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/ForFileIO.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@BindingAnnotation +public @interface ForFileIO +{ +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java index fe9913361d30..caf5d9996daf 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java @@ -79,6 +79,10 @@ public class IcebergConfig private boolean sortedWritingEnabled = true; private boolean queryPartitionFilterRequired; private int splitManagerThreads = Runtime.getRuntime().availableProcessors() * 2; + private boolean manifestCachingEnabled; + private Duration manifestCachingTTL = new Duration(60, SECONDS); + private DataSize manifestCachingMaxFileSize = DataSize.of(8, MEGABYTE); + private DataSize manifestCachingMaxTotalSize = DataSize.of(100, MEGABYTE); public CatalogType getCatalogType() { @@ -433,4 +437,52 @@ public boolean isStorageSchemaSetWhenHidingIsEnabled() { return hideMaterializedViewStorageTable && materializedViewsStorageSchema.isPresent(); } + + public boolean isManifestCachingEnabled() + { + return manifestCachingEnabled; + } + + @Config("iceberg.manifest-caching.enabled") + @ConfigDescription("Enable in-memory caching for manifest files") + public void setManifestCachingEnabled(boolean manifestCachingEnabled) + { + this.manifestCachingEnabled = manifestCachingEnabled; + } + + public Duration getManifestCachingTTL() + { + return manifestCachingTTL; + } + + @Config("iceberg.manifest-caching.ttl") + @ConfigDescription("Maximum duration for which an entry stays in the manifest cache") + public void setManifestCachingTTL(Duration manifestCachingTTL) + { + this.manifestCachingTTL = manifestCachingTTL; + } + + public DataSize getManifestCachingMaxFileSize() + { + return manifestCachingMaxFileSize; + } + + @Config("iceberg.manifest-caching.max-file-size") + @ConfigDescription("Maximum size of a manifest file to be considered for caching") + public void setManifestCachingMaxFileSize(DataSize manifestCachingMaxFileSize) + { + this.manifestCachingMaxFileSize = manifestCachingMaxFileSize; + } + + public DataSize getManifestCachingMaxTotalSize() + { + return manifestCachingMaxTotalSize; + } + + @Config("iceberg.manifest-caching.max-total-size") + @ConfigDescription("Maximum total amount of bytes to cache in manifest cache") + public void setManifestCachingMaxTotalSize(DataSize manifestCachingMaxTotalSize) + { + this.manifestCachingMaxTotalSize = manifestCachingMaxTotalSize; + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java index 60e694487f29..209559be4379 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java @@ -93,7 +93,7 @@ public static Connector createConnector( new MBeanModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin.iceberg", "trino.plugin.iceberg"), new JsonModule(), - new IcebergModule(), + new IcebergModule(context.getNodeManager()), new IcebergSecurityModule(), icebergCatalogModule.orElse(new IcebergCatalogModule()), new MBeanServerModule(), diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java index dcf307c2e21e..8a616b6f74af 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergModule.java @@ -13,12 +13,16 @@ */ package io.trino.plugin.iceberg; +import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; +import com.google.inject.Inject; import com.google.inject.Key; import com.google.inject.Module; +import com.google.inject.Provider; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.Multibinder; import io.trino.filesystem.cache.AllowFilesystemCacheOnCoordinator; import io.trino.filesystem.cache.CacheKeyProvider; @@ -42,6 +46,7 @@ import io.trino.plugin.iceberg.procedure.RegisterTableProcedure; import io.trino.plugin.iceberg.procedure.RemoveOrphanFilesTableProcedure; import io.trino.plugin.iceberg.procedure.UnregisterTableProcedure; +import io.trino.spi.NodeManager; import io.trino.spi.connector.ConnectorNodePartitioningProvider; import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; @@ -50,7 +55,9 @@ import io.trino.spi.function.FunctionProvider; import io.trino.spi.function.table.ConnectorTableFunction; import io.trino.spi.procedure.Procedure; +import org.apache.iceberg.CatalogProperties; +import java.util.Map; import java.util.concurrent.ExecutorService; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; @@ -59,12 +66,20 @@ import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.configuration.ConfigBinder.configBinder; import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; import static org.weakref.jmx.guice.ExportBinder.newExporter; public class IcebergModule implements Module { + private final NodeManager nodeManager; + + public IcebergModule(NodeManager nodeManager) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + } + @Override public void configure(Binder binder) { @@ -119,6 +134,13 @@ public void configure(Binder binder) newOptionalBinder(binder, IcebergFileSystemFactory.class).setDefault().to(DefaultIcebergFileSystemFactory.class).in(Scopes.SINGLETON); newOptionalBinder(binder, CacheKeyProvider.class).setBinding().to(IcebergCacheKeyProvider.class).in(Scopes.SINGLETON); newOptionalBinder(binder, Key.get(boolean.class, AllowFilesystemCacheOnCoordinator.class)).setBinding().toInstance(true); + + if (nodeManager.getCurrentNode().isCoordinator()) { + newOptionalBinder(binder, Key.get(new TypeLiteral>() {}, ForFileIO.class)).setBinding().toProvider(CoordinatorFileIOPropertiesProvider.class).in(Scopes.SINGLETON); + } + else { + newOptionalBinder(binder, Key.get(new TypeLiteral>() {}, ForFileIO.class)).setBinding().toInstance(Map.of()); + } } @Provides @@ -133,4 +155,30 @@ public ExecutorService createSplitManagerExecutor(CatalogName catalogName, Icebe config.getSplitManagerThreads(), daemonThreadsNamed("iceberg-split-manager-" + catalogName + "-%s")); } + + public static class CoordinatorFileIOPropertiesProvider + implements Provider> + { + private final IcebergConfig icebergConfig; + + @Inject + public CoordinatorFileIOPropertiesProvider(IcebergConfig icebergConfig) + { + this.icebergConfig = requireNonNull(icebergConfig, "icebergConfig is null"); + } + + @Override + public Map get() + { + if (icebergConfig.isManifestCachingEnabled()) { + return ImmutableMap.builder() + .put(CatalogProperties.IO_MANIFEST_CACHE_ENABLED, "true") + .put(CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, Long.toString(icebergConfig.getManifestCachingTTL().toMillis())) + .put(CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, Long.toString(icebergConfig.getManifestCachingMaxFileSize().toBytes())) + .put(CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, Long.toString(icebergConfig.getManifestCachingMaxTotalSize().toBytes())) + .buildOrThrow(); + } + return Map.of(); + } + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperationsProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperationsProvider.java index b14f952b0e94..f628acbf767d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperationsProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/FileMetastoreTableOperationsProvider.java @@ -15,6 +15,7 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.catalog.IcebergTableOperations; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -22,6 +23,7 @@ import io.trino.plugin.iceberg.fileio.ForwardingFileIo; import io.trino.spi.connector.ConnectorSession; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -30,11 +32,13 @@ public class FileMetastoreTableOperationsProvider implements IcebergTableOperationsProvider { private final TrinoFileSystemFactory fileSystemFactory; + private final Map fileIoProperties; @Inject - public FileMetastoreTableOperationsProvider(TrinoFileSystemFactory fileSystemFactory) + public FileMetastoreTableOperationsProvider(TrinoFileSystemFactory fileSystemFactory, @ForFileIO Map fileIoProperties) { this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -47,7 +51,7 @@ public IcebergTableOperations createTableOperations( Optional location) { return new FileMetastoreTableOperations( - new ForwardingFileIo(fileSystemFactory.create(session)), + new ForwardingFileIo(fileSystemFactory.create(session), fileIoProperties), ((TrinoHiveCatalog) catalog).getMetastore(), session, database, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperationsProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperationsProvider.java index 4b54259edf7a..13856ab56911 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperationsProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperationsProvider.java @@ -17,6 +17,7 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.catalog.IcebergTableOperations; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -24,6 +25,7 @@ import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.TypeManager; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -36,6 +38,7 @@ public class GlueIcebergTableOperationsProvider private final TrinoFileSystemFactory fileSystemFactory; private final AWSGlueAsync glueClient; private final GlueMetastoreStats stats; + private final Map fileIoProperties; @Inject public GlueIcebergTableOperationsProvider( @@ -43,13 +46,15 @@ public GlueIcebergTableOperationsProvider( IcebergGlueCatalogConfig catalogConfig, TrinoFileSystemFactory fileSystemFactory, GlueMetastoreStats stats, - AWSGlueAsync glueClient) + AWSGlueAsync glueClient, + @ForFileIO Map fileIoProperties) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.cacheTableMetadata = catalogConfig.isCacheTableMetadata(); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.stats = requireNonNull(stats, "stats is null"); this.glueClient = requireNonNull(glueClient, "glueClient is null"); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -69,7 +74,7 @@ public IcebergTableOperations createTableOperations( // Share Glue Table cache between Catalog and TableOperations so that, when doing metadata queries (e.g. information_schema.columns) // the GetTableRequest is issued once per table. ((TrinoGlueCatalog) catalog)::getTable, - new ForwardingFileIo(fileSystemFactory.create(session)), + new ForwardingFileIo(fileSystemFactory.create(session), fileIoProperties), session, database, table, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 4101b4af946b..8b7ba41172bd 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -51,6 +51,7 @@ import io.trino.plugin.hive.ViewAlreadyExistsException; import io.trino.plugin.hive.ViewReaderUtil; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergMaterializedViewDefinition; import io.trino.plugin.iceberg.IcebergMetadata; import io.trino.plugin.iceberg.UnknownTableTypeException; @@ -182,6 +183,7 @@ public class TrinoGlueCatalog private final GlueMetastoreStats stats; private final boolean hideMaterializedViewStorageTable; private final boolean isUsingSystemSecurity; + private final Map fileIoProperties; private final Cache glueTableCache = EvictableCacheBuilder.newBuilder() // Even though this is query-scoped, this still needs to be bounded. information_schema queries can access large number of tables. @@ -210,7 +212,8 @@ public TrinoGlueCatalog( boolean isUsingSystemSecurity, Optional defaultSchemaLocation, boolean useUniqueTableLocation, - boolean hideMaterializedViewStorageTable) + boolean hideMaterializedViewStorageTable, + @ForFileIO Map fileIoProperties) { super(catalogName, typeManager, tableOperationsProvider, fileSystemFactory, useUniqueTableLocation); this.trinoVersion = requireNonNull(trinoVersion, "trinoVersion is null"); @@ -222,6 +225,7 @@ public TrinoGlueCatalog( this.isUsingSystemSecurity = isUsingSystemSecurity; this.defaultSchemaLocation = requireNonNull(defaultSchemaLocation, "defaultSchemaLocation is null"); this.hideMaterializedViewStorageTable = hideMaterializedViewStorageTable; + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -852,7 +856,7 @@ private Optional getTableAndCacheMetada try { // Cache the TableMetadata while we have the Table retrieved anyway // Note: this is racy from cache invalidation perspective, but it should not matter here - uncheckedCacheGet(tableMetadataCache, schemaTableName, () -> TableMetadataParser.read(new ForwardingFileIo(fileSystemFactory.create(session)), metadataLocation)); + uncheckedCacheGet(tableMetadataCache, schemaTableName, () -> TableMetadataParser.read(new ForwardingFileIo(fileSystemFactory.create(session), fileIoProperties), metadataLocation)); } catch (RuntimeException e) { LOG.warn(e, "Failed to cache table metadata from table at %s", metadataLocation); @@ -1381,7 +1385,7 @@ private TableMetadata getMaterializedViewTableMetadata(ConnectorSession session, requireNonNull(storageMetadataLocation, "storageMetadataLocation is null"); return uncheckedCacheGet(tableMetadataCache, storageTableName, () -> { TrinoFileSystem fileSystem = fileSystemFactory.create(session); - return TableMetadataParser.read(new ForwardingFileIo(fileSystem), storageMetadataLocation); + return TableMetadataParser.read(new ForwardingFileIo(fileSystem, fileIoProperties), storageMetadataLocation); }); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java index 89e844d7ac47..7dda5d0c4cb3 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java @@ -20,6 +20,7 @@ import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergSecurityConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; @@ -30,6 +31,7 @@ import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; +import java.util.Map; import java.util.Optional; import static io.trino.plugin.iceberg.IcebergSecurityConfig.IcebergSecurity.SYSTEM; @@ -50,6 +52,7 @@ public class TrinoGlueCatalogFactory private final boolean hideMaterializedViewStorageTable; private final GlueMetastoreStats stats; private final boolean isUsingSystemSecurity; + private final Map fileIoProperties; @Inject public TrinoGlueCatalogFactory( @@ -63,7 +66,8 @@ public TrinoGlueCatalogFactory( IcebergGlueCatalogConfig catalogConfig, IcebergSecurityConfig securityConfig, GlueMetastoreStats stats, - AWSGlueAsync glueClient) + AWSGlueAsync glueClient, + @ForFileIO Map fileIoProperties) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); @@ -77,6 +81,7 @@ public TrinoGlueCatalogFactory( this.hideMaterializedViewStorageTable = icebergConfig.isHideMaterializedViewStorageTable(); this.stats = requireNonNull(stats, "stats is null"); this.isUsingSystemSecurity = securityConfig.getSecuritySystem() == SYSTEM; + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Managed @@ -101,6 +106,7 @@ public TrinoCatalog create(ConnectorIdentity identity) isUsingSystemSecurity, defaultSchemaLocation, isUniqueTableLocation, - hideMaterializedViewStorageTable); + hideMaterializedViewStorageTable, + fileIoProperties); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperationsProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperationsProvider.java index b7a31bdbe817..ac30c33f1b25 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperationsProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/HiveMetastoreTableOperationsProvider.java @@ -16,12 +16,14 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreFactory; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.catalog.IcebergTableOperations; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.fileio.ForwardingFileIo; import io.trino.spi.connector.ConnectorSession; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -31,12 +33,14 @@ public class HiveMetastoreTableOperationsProvider { private final TrinoFileSystemFactory fileSystemFactory; private final ThriftMetastoreFactory thriftMetastoreFactory; + private final Map fileIoProperties; @Inject - public HiveMetastoreTableOperationsProvider(TrinoFileSystemFactory fileSystemFactory, ThriftMetastoreFactory thriftMetastoreFactory) + public HiveMetastoreTableOperationsProvider(TrinoFileSystemFactory fileSystemFactory, ThriftMetastoreFactory thriftMetastoreFactory, @ForFileIO Map fileIoProperties) { this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.thriftMetastoreFactory = requireNonNull(thriftMetastoreFactory, "thriftMetastoreFactory is null"); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -49,7 +53,7 @@ public IcebergTableOperations createTableOperations( Optional location) { return new HiveMetastoreTableOperations( - new ForwardingFileIo(fileSystemFactory.create(session)), + new ForwardingFileIo(fileSystemFactory.create(session), fileIoProperties), ((TrinoHiveCatalog) catalog).getMetastore(), thriftMetastoreFactory.createMetastore(Optional.of(session.getIdentity())), session, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java index 616e77a97071..1a36c4eb9686 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java @@ -33,6 +33,7 @@ import io.trino.plugin.hive.metastore.PrincipalPrivileges; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.util.HiveUtil; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergTableName; import io.trino.plugin.iceberg.UnknownTableTypeException; import io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog; @@ -142,6 +143,7 @@ public class TrinoHiveCatalog private final boolean isUsingSystemSecurity; private final boolean deleteSchemaLocationsFallback; private final boolean hideMaterializedViewStorageTable; + private final Map fileIoProperties; private final Cache tableMetadataCache = EvictableCacheBuilder.newBuilder() .maximumSize(PER_QUERY_CACHE_SIZE) @@ -157,7 +159,8 @@ public TrinoHiveCatalog( boolean useUniqueTableLocation, boolean isUsingSystemSecurity, boolean deleteSchemaLocationsFallback, - boolean hideMaterializedViewStorageTable) + boolean hideMaterializedViewStorageTable, + @ForFileIO Map fileIoProperties) { super(catalogName, typeManager, tableOperationsProvider, fileSystemFactory, useUniqueTableLocation); this.metastore = requireNonNull(metastore, "metastore is null"); @@ -166,6 +169,7 @@ public TrinoHiveCatalog( this.isUsingSystemSecurity = isUsingSystemSecurity; this.deleteSchemaLocationsFallback = deleteSchemaLocationsFallback; this.hideMaterializedViewStorageTable = hideMaterializedViewStorageTable; + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } public CachingHiveMetastore getMetastore() @@ -731,7 +735,7 @@ public void dropMaterializedView(ConnectorSession session, SchemaTableName viewN checkState(storageMetadataLocation != null, "Storage location missing in definition of materialized view " + viewName); TrinoFileSystem fileSystem = fileSystemFactory.create(session); - TableMetadata metadata = TableMetadataParser.read(new ForwardingFileIo(fileSystem), storageMetadataLocation); + TableMetadata metadata = TableMetadataParser.read(new ForwardingFileIo(fileSystem, fileIoProperties), storageMetadataLocation); String storageLocation = metadata.location(); try { fileSystem.deleteDirectory(Location.of(storageLocation)); @@ -821,7 +825,7 @@ private TableMetadata getMaterializedViewTableMetadata(ConnectorSession session, String storageMetadataLocation = materializedView.getParameters().get(METADATA_LOCATION_PROP); checkState(storageMetadataLocation != null, "Storage location missing in definition of materialized view " + materializedView.getTableName()); TrinoFileSystem fileSystem = fileSystemFactory.create(session); - return TableMetadataParser.read(new ForwardingFileIo(fileSystem), storageMetadataLocation); + return TableMetadataParser.read(new ForwardingFileIo(fileSystem, fileIoProperties), storageMetadataLocation); }); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java index 252f0cccb087..87d6ffd8c0db 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java @@ -20,6 +20,7 @@ import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergSecurityConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; @@ -28,6 +29,7 @@ import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; +import java.util.Map; import java.util.Optional; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; @@ -48,6 +50,7 @@ public class TrinoHiveCatalogFactory private final boolean isUsingSystemSecurity; private final boolean deleteSchemaLocationsFallback; private final boolean hideMaterializedViewStorageTable; + private final Map fileIoProperties; @Inject public TrinoHiveCatalogFactory( @@ -58,7 +61,8 @@ public TrinoHiveCatalogFactory( TypeManager typeManager, IcebergTableOperationsProvider tableOperationsProvider, NodeVersion nodeVersion, - IcebergSecurityConfig securityConfig) + IcebergSecurityConfig securityConfig, + @ForFileIO Map fileIoProperties) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.metastoreFactory = requireNonNull(metastoreFactory, "metastoreFactory is null"); @@ -70,6 +74,7 @@ public TrinoHiveCatalogFactory( this.isUsingSystemSecurity = securityConfig.getSecuritySystem() == SYSTEM; this.deleteSchemaLocationsFallback = config.isDeleteSchemaLocationsFallback(); this.hideMaterializedViewStorageTable = config.isHideMaterializedViewStorageTable(); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -86,6 +91,7 @@ public TrinoCatalog create(ConnectorIdentity identity) isUniqueTableLocation, isUsingSystemSecurity, deleteSchemaLocationsFallback, - hideMaterializedViewStorageTable); + hideMaterializedViewStorageTable, + fileIoProperties); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/IcebergJdbcTableOperationsProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/IcebergJdbcTableOperationsProvider.java index ad8c861afc08..78a2fdfee986 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/IcebergJdbcTableOperationsProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/IcebergJdbcTableOperationsProvider.java @@ -15,12 +15,14 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.catalog.IcebergTableOperations; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.fileio.ForwardingFileIo; import io.trino.spi.connector.ConnectorSession; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -30,12 +32,14 @@ public class IcebergJdbcTableOperationsProvider { private final TrinoFileSystemFactory fileSystemFactory; private final IcebergJdbcClient jdbcClient; + private final Map fileIoProperties; @Inject - public IcebergJdbcTableOperationsProvider(IcebergJdbcClient jdbcClient, TrinoFileSystemFactory fileSystemFactory) + public IcebergJdbcTableOperationsProvider(IcebergJdbcClient jdbcClient, TrinoFileSystemFactory fileSystemFactory, @ForFileIO Map fileIoProperties) { this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -48,7 +52,7 @@ public IcebergTableOperations createTableOperations( Optional location) { return new IcebergJdbcTableOperations( - new ForwardingFileIo(fileSystemFactory.create(session)), + new ForwardingFileIo(fileSystemFactory.create(session), fileIoProperties), jdbcClient, session, database, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalogFactory.java index 5c887f50e48c..b7cf207b2618 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/jdbc/TrinoJdbcCatalogFactory.java @@ -17,6 +17,7 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.base.CatalogName; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -48,6 +49,7 @@ public class TrinoJdbcCatalogFactory private final boolean isUniqueTableLocation; private final Map catalogProperties; private final JdbcClientPool clientPool; + private final Map fileIoProperties; @Inject public TrinoJdbcCatalogFactory( @@ -57,7 +59,8 @@ public TrinoJdbcCatalogFactory( TrinoFileSystemFactory fileSystemFactory, IcebergJdbcClient jdbcClient, IcebergJdbcCatalogConfig jdbcConfig, - IcebergConfig icebergConfig) + IcebergConfig icebergConfig, + @ForFileIO Map fileIoProperties) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); @@ -67,6 +70,7 @@ public TrinoJdbcCatalogFactory( this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); this.jdbcCatalogName = jdbcConfig.getCatalogName(); this.defaultWarehouseDir = jdbcConfig.getDefaultWarehouseDir(); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); ImmutableMap.Builder properties = ImmutableMap.builder(); properties.put(URI, jdbcConfig.getConnectionUrl()); @@ -88,7 +92,7 @@ public void shutdown() public TrinoCatalog create(ConnectorIdentity identity) { JdbcCatalog jdbcCatalog = new JdbcCatalog( - config -> new ForwardingFileIo(fileSystemFactory.create(identity)), + config -> new ForwardingFileIo(fileSystemFactory.create(identity), fileIoProperties), config -> clientPool, false); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperationsProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperationsProvider.java index 0be1d7df9c68..953024abb4b5 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperationsProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperationsProvider.java @@ -15,6 +15,7 @@ import com.google.inject.Inject; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.catalog.IcebergTableOperations; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -22,6 +23,7 @@ import io.trino.spi.connector.ConnectorSession; import org.apache.iceberg.nessie.NessieIcebergClient; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -31,12 +33,14 @@ public class IcebergNessieTableOperationsProvider { private final TrinoFileSystemFactory fileSystemFactory; private final NessieIcebergClient nessieClient; + private final Map fileIoProperties; @Inject - public IcebergNessieTableOperationsProvider(TrinoFileSystemFactory fileSystemFactory, NessieIcebergClient nessieClient) + public IcebergNessieTableOperationsProvider(TrinoFileSystemFactory fileSystemFactory, NessieIcebergClient nessieClient, @ForFileIO Map fileIoProperties) { this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.nessieClient = requireNonNull(nessieClient, "nessieClient is null"); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -50,7 +54,7 @@ public IcebergTableOperations createTableOperations( { return new IcebergNessieTableOperations( nessieClient, - new ForwardingFileIo(fileSystemFactory.create(session)), + new ForwardingFileIo(fileSystemFactory.create(session), fileIoProperties), session, database, table, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java index fd817a53ec8f..f6ce051021fc 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java @@ -18,6 +18,7 @@ import com.google.inject.Inject; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.NodeVersion; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergFileSystemFactory; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -30,6 +31,7 @@ import org.apache.iceberg.rest.RESTSessionCatalog; import java.net.URI; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -46,6 +48,7 @@ public class TrinoIcebergRestCatalogFactory private final boolean vendedCredentialsEnabled; private final SecurityProperties securityProperties; private final boolean uniqueTableLocation; + private final Map fileIoProperties; @GuardedBy("this") private RESTSessionCatalog icebergCatalog; @@ -57,7 +60,8 @@ public TrinoIcebergRestCatalogFactory( IcebergRestCatalogConfig restConfig, SecurityProperties securityProperties, IcebergConfig icebergConfig, - NodeVersion nodeVersion) + NodeVersion nodeVersion, + @ForFileIO Map fileIoProperties) { this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.catalogName = requireNonNull(catalogName, "catalogName is null"); @@ -70,6 +74,7 @@ public TrinoIcebergRestCatalogFactory( this.securityProperties = requireNonNull(securityProperties, "securityProperties is null"); requireNonNull(icebergConfig, "icebergConfig is null"); this.uniqueTableLocation = icebergConfig.isUniqueTableLocation(); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -94,7 +99,11 @@ public synchronized TrinoCatalog create(ConnectorIdentity identity) ConnectorIdentity currentIdentity = (context.wrappedIdentity() != null) ? ((ConnectorIdentity) context.wrappedIdentity()) : ConnectorIdentity.ofUser("fake"); - return new ForwardingFileIo(fileSystemFactory.create(currentIdentity, config), config); + Map combinedFileIoProperties = ImmutableMap.builder() + .putAll(config) + .putAll(fileIoProperties) + .buildOrThrow(); + return new ForwardingFileIo(fileSystemFactory.create(currentIdentity, combinedFileIoProperties), config); }); icebergCatalogInstance.initialize(catalogName.toString(), properties.buildOrThrow()); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RegisterTableProcedure.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RegisterTableProcedure.java index a3b3efb385fb..5b772e164607 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RegisterTableProcedure.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/RegisterTableProcedure.java @@ -21,6 +21,7 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.plugin.iceberg.ForFileIO; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; @@ -37,6 +38,7 @@ import java.lang.invoke.MethodHandle; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.google.common.collect.Iterables.getOnlyElement; @@ -80,13 +82,19 @@ public class RegisterTableProcedure private final TrinoCatalogFactory catalogFactory; private final TrinoFileSystemFactory fileSystemFactory; private final boolean registerTableProcedureEnabled; + private final Map fileIoProperties; @Inject - public RegisterTableProcedure(TrinoCatalogFactory catalogFactory, TrinoFileSystemFactory fileSystemFactory, IcebergConfig icebergConfig) + public RegisterTableProcedure( + TrinoCatalogFactory catalogFactory, + TrinoFileSystemFactory fileSystemFactory, + IcebergConfig icebergConfig, + @ForFileIO Map fileIoProperties) { this.catalogFactory = requireNonNull(catalogFactory, "catalogFactory is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.registerTableProcedureEnabled = requireNonNull(icebergConfig, "icebergConfig is null").isRegisterTableProcedureEnabled(); + this.fileIoProperties = requireNonNull(fileIoProperties, "fileIoProperties is null"); } @Override @@ -147,7 +155,7 @@ private void doRegisterTable( TableMetadata tableMetadata; try { // Try to read the metadata file. Invalid metadata file will throw the exception. - tableMetadata = TableMetadataParser.read(new ForwardingFileIo(fileSystem), metadataLocation); + tableMetadata = TableMetadataParser.read(new ForwardingFileIo(fileSystem, fileIoProperties), metadataLocation); } catch (RuntimeException e) { throw new TrinoException(ICEBERG_INVALID_METADATA, "Invalid metadata file: " + metadataLocation, e); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java index e4572542c257..32f0443cee6b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java @@ -32,6 +32,7 @@ import org.apache.iceberg.Table; import org.junit.jupiter.api.Test; +import java.util.Map; import java.util.Optional; import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.createPerTransactionCache; @@ -55,7 +56,7 @@ protected QueryRunner createQueryRunner() .createMetastore(Optional.empty()); CachingHiveMetastore cachingHiveMetastore = createPerTransactionCache(metastore, 1000); TrinoFileSystemFactory fileSystemFactory = getFileSystemFactory(queryRunner); - tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory); + tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory, Map.of()); trinoCatalog = new TrinoHiveCatalog( new CatalogName("catalog"), cachingHiveMetastore, @@ -66,7 +67,8 @@ protected QueryRunner createQueryRunner() false, false, false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); return queryRunner; } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java index 6522c0889ea4..0c1b041222d8 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java @@ -85,7 +85,7 @@ protected QueryRunner createQueryRunner() queryRunner.createCatalog(ICEBERG_CATALOG, "iceberg", ImmutableMap.of("iceberg.file-format", "ORC")); TrinoFileSystemFactory fileSystemFactory = getFileSystemFactory(queryRunner); - tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory); + tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory, Map.of()); HiveMetastore metastore = ((IcebergConnector) queryRunner.getCoordinator().getConnector(ICEBERG_CATALOG)).getInjector() .getInstance(HiveMetastoreFactory.class) @@ -102,7 +102,8 @@ protected QueryRunner createQueryRunner() false, false, false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java index 7b09114f2e23..c09209317349 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java @@ -122,11 +122,12 @@ protected QueryRunner createQueryRunner() new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), - new FileMetastoreTableOperationsProvider(fileSystemFactory), + new FileMetastoreTableOperationsProvider(fileSystemFactory, Map.of()), false, false, false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); return queryRunner; } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index d89aef8d1a57..224d558f77d5 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -1001,7 +1001,7 @@ private Table updateTableToV2(String tableName) private BaseTable loadTable(String tableName) { - IcebergTableOperationsProvider tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory); + IcebergTableOperationsProvider tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory, Map.of()); CachingHiveMetastore cachingHiveMetastore = createPerTransactionCache(metastore, 1000); TrinoCatalog catalog = new TrinoHiveCatalog( new CatalogName("hive"), @@ -1013,7 +1013,8 @@ private BaseTable loadTable(String tableName) false, false, false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); return (BaseTable) loadIcebergTable(catalog, tableOperationsProvider, SESSION, new SchemaTableName("tpch", tableName)); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java index 474fd270a7bc..b08b82ea51e9 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Map; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; @@ -79,10 +80,11 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), - new FileMetastoreTableOperationsProvider(fileSystemFactory), + new FileMetastoreTableOperationsProvider(fileSystemFactory, Map.of()), useUniqueTableLocations, false, false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java index 55ab9868a14d..93f38ad1eedf 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java @@ -88,14 +88,16 @@ private TrinoCatalog createGlueTrinoCatalog(boolean useUniqueTableLocations, boo catalogConfig, HDFS_FILE_SYSTEM_FACTORY, new GlueMetastoreStats(), - glueClient), + glueClient, + Map.of()), "test", glueClient, new GlueMetastoreStats(), useSystemSecurity, Optional.empty(), useUniqueTableLocations, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); } /** @@ -217,14 +219,16 @@ public void testDefaultLocation() catalogConfig, fileSystemFactory, new GlueMetastoreStats(), - glueClient), + glueClient, + Map.of()), "test", glueClient, new GlueMetastoreStats(), false, Optional.of(tmpDirectory.toAbsolutePath().toString()), false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); String namespace = "test_default_location_" + randomNameSuffix(); String table = "tableName"; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index 03adafadfcff..bcc091f2cb4a 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -116,25 +116,29 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) new TrinoViewHiveMetastore(metastore, false, "trino-version", "Test"), fileSystemFactory, new TestingTypeManager(), - new HiveMetastoreTableOperationsProvider(fileSystemFactory, new ThriftMetastoreFactory() - { - @Override - public boolean isImpersonationEnabled() - { - verify(new ThriftMetastoreConfig().isImpersonationEnabled(), "This test wants to test the default behavior and assumes it's off"); - return false; - } + new HiveMetastoreTableOperationsProvider( + fileSystemFactory, + new ThriftMetastoreFactory() + { + @Override + public boolean isImpersonationEnabled() + { + verify(new ThriftMetastoreConfig().isImpersonationEnabled(), "This test wants to test the default behavior and assumes it's off"); + return false; + } - @Override - public ThriftMetastore createMetastore(Optional identity) - { - return thriftMetastore; - } - }), + @Override + public ThriftMetastore createMetastore(Optional identity) + { + return thriftMetastore; + } + }, + Map.of()), useUniqueTableLocations, false, false, - new IcebergConfig().isHideMaterializedViewStorageTable()); + new IcebergConfig().isHideMaterializedViewStorageTable(), + Map.of()); } @Override diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java index b9a64fe3eba8..b1326cef6dcb 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java @@ -102,7 +102,7 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) new CatalogName("catalog_name"), new TestingTypeManager(), fileSystemFactory, - new IcebergNessieTableOperationsProvider(fileSystemFactory, nessieClient), + new IcebergNessieTableOperationsProvider(fileSystemFactory, nessieClient, Map.of()), nessieClient, tmpDirectory.toAbsolutePath().toString(), useUniqueTableLocations); @@ -126,7 +126,7 @@ public void testDefaultLocation() new CatalogName("catalog_name"), new TestingTypeManager(), fileSystemFactory, - new IcebergNessieTableOperationsProvider(fileSystemFactory, nessieClient), + new IcebergNessieTableOperationsProvider(fileSystemFactory, nessieClient, Map.of()), nessieClient, icebergNessieCatalogConfig.getDefaultWarehouseDir(), false);