diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index aa220fc1ee100..9d6644644ae50 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -411,10 +411,6 @@ Manifest File Caching As of Iceberg version 1.1.0, Apache Iceberg provides a mechanism to cache the contents of Iceberg manifest files in memory. This feature helps to reduce repeated reads of small Iceberg manifest files from remote storage. -.. note:: - - Currently, manifest file caching is supported for Hadoop and Nessie catalogs in the Presto Iceberg connector. - The following configuration properties are available: ==================================================== ============================================================= ============ diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index a909b7f5143cf..8f9e242f74087 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -12,7 +12,7 @@ ${project.parent.basedir} - 1.5.0 + 1.6.0-SNAPSHOT 0.77.1 @@ -243,6 +243,31 @@ org.apache.iceberg iceberg-bundled-guava ${dep.iceberg.version} + + + + org.apache.iceberg + iceberg-common + ${dep.iceberg.version} + runtime + + + + com.github.ben-manes.caffeine + caffeine + 2.9.3 + + + + org.apache.hudi + hudi-presto-bundle + runtime + + + + com.google.errorprone + error_prone_annotations + 2.10.0 runtime diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HdfsFileIO.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HdfsFileIO.java index 3eaebcf7b0a36..ebb73c0a3c942 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HdfsFileIO.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HdfsFileIO.java @@ -16,14 +16,18 @@ import com.facebook.presto.hive.HdfsContext; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.fs.Path; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.util.SerializableMap; import java.io.IOException; +import java.util.Map; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; +import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; public class HdfsFileIO @@ -31,6 +35,7 @@ public class HdfsFileIO { private final HdfsEnvironment environment; private final HdfsContext context; + private SerializableMap properties = SerializableMap.copyOf(ImmutableMap.of()); public HdfsFileIO(HdfsEnvironment environment, HdfsContext context) { @@ -38,6 +43,18 @@ public HdfsFileIO(HdfsEnvironment environment, HdfsContext context) this.context = requireNonNull(context, "context is null"); } + @Override + public void initialize(Map props) + { + this.properties = SerializableMap.copyOf(props); + } + + @Override + public Map properties() + { + return properties.immutableMap(); + } + @Override public InputFile newInputFile(String path) { @@ -61,4 +78,11 @@ public void deleteFile(String pathString) throw new PrestoException(ICEBERG_FILESYSTEM_ERROR, "Failed to delete file: " + path, e); } } + + @Override + public String toString() + { + return toStringHelper(this) + .toString(); + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java index e5d8bc46d9526..2e9137c6712bb 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java @@ -68,6 +68,7 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.isPrestoView; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA; import static com.facebook.presto.iceberg.IcebergUtil.isIcebergTable; +import static com.facebook.presto.iceberg.IcebergUtil.loadCachingProperties; import static com.facebook.presto.iceberg.IcebergUtil.toHiveColumns; import static com.facebook.presto.spi.security.PrincipalType.USER; import static com.google.common.base.Preconditions.checkState; @@ -100,6 +101,7 @@ public class HiveTableOperations private final ExtendedHiveMetastore metastore; private final MetastoreContext metastoreContext; + private final IcebergConfig icebergConfig; private final String database; private final String tableName; private final Optional owner; @@ -120,6 +122,7 @@ public HiveTableOperations( HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, IcebergHiveTableOperationsConfig config, + IcebergConfig icebergConfig, String database, String table) { @@ -127,6 +130,7 @@ public HiveTableOperations( metastore, metastoreContext, config, + icebergConfig, database, table, Optional.empty(), @@ -139,6 +143,7 @@ public HiveTableOperations( HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, IcebergHiveTableOperationsConfig config, + IcebergConfig icebergConfig, String database, String table, String owner, @@ -148,6 +153,7 @@ public HiveTableOperations( metastore, metastoreContext, config, + icebergConfig, database, table, Optional.of(requireNonNull(owner, "owner is null")), @@ -159,6 +165,7 @@ private HiveTableOperations( ExtendedHiveMetastore metastore, MetastoreContext metastoreContext, IcebergHiveTableOperationsConfig config, + IcebergConfig icebergConfig, String database, String table, Optional owner, @@ -167,6 +174,7 @@ private HiveTableOperations( this.fileIO = requireNonNull(fileIO, "fileIO is null"); this.metastore = requireNonNull(metastore, "metastore is null"); this.metastoreContext = requireNonNull(metastoreContext, "metastore context is null"); + this.icebergConfig = requireNonNull(icebergConfig, "icebergConfig context is null"); this.database = requireNonNull(database, "database is null"); this.tableName = requireNonNull(table, "table is null"); this.owner = requireNonNull(owner, "owner is null"); @@ -338,6 +346,11 @@ public void commit(@Nullable TableMetadata base, TableMetadata metadata) @Override public FileIO io() { + if (icebergConfig.getManifestCachingEnabled()) { + Map properties = new HashMap<>(); + loadCachingProperties(properties, icebergConfig); + fileIO.initialize(properties); + } return fileIO; } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java index 66d06f3a31061..b51ceee15b46a 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergCommonModule.java @@ -168,6 +168,17 @@ public void setup(Binder binder) configBinder(binder).bindConfig(ParquetCacheConfig.class, connectorId); binder.bind(ConnectorPlanOptimizerProvider.class).to(IcebergPlanOptimizerProvider.class).in(Scopes.SINGLETON); + + boolean isManifestFileCachingEnabled = buildConfigObject(IcebergConfig.class).getManifestCachingEnabled(); + if (isManifestFileCachingEnabled) { + binder.bind(PrestoInMemoryContentCacheManager.class).in(Scopes.SINGLETON); + binder.requestStaticInjection(PrestoInMemoryContentCacheManager.class); +// newExporter(binder).export(PrestoInMemoryContentCacheManager.class).withGeneratedName(); + + binder.bind(IcebergFileIOStats.class).in(Scopes.SINGLETON); + binder.requestStaticInjection(IcebergFileIOStats.class); + newExporter(binder).export(IcebergFileIOStats.class).withGeneratedName(); + } } @ForCachingHiveMetastore diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java index 643906667a507..418c09b19f25c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergConfig.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.statistics.ColumnStatisticType; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import org.apache.iceberg.hadoop.HadoopFileIO; import javax.validation.constraints.DecimalMax; import javax.validation.constraints.DecimalMin; @@ -33,9 +32,7 @@ import static com.facebook.presto.iceberg.CatalogType.HIVE; import static com.facebook.presto.iceberg.IcebergFileFormat.PARQUET; import static com.facebook.presto.iceberg.util.StatisticsUtil.decodeMergeFlags; -import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT; import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT; -import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT; import static org.apache.iceberg.TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT; import static org.apache.iceberg.TableProperties.METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT; @@ -59,11 +56,12 @@ public class IcebergConfig private boolean metadataDeleteAfterCommit = METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT; private EnumSet hiveStatisticsMergeFlags = EnumSet.noneOf(ColumnStatisticType.class); - private String fileIOImpl = HadoopFileIO.class.getName(); - private boolean manifestCachingEnabled; - private long maxManifestCacheSize = IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT; - private long manifestCacheExpireDuration = IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT; + private String fileIOImpl = HdfsFileIO.class.getName(); + private boolean manifestCachingEnabled = true; + private long maxManifestCacheSize = 1073741824; // 1 GB, default 100 MB + private long manifestCacheExpireDuration = 86400000; // 24 hrs private long manifestCacheMaxContentLength = IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT; + private String fileIOContentCacheManager = PrestoInMemoryContentCacheManager.class.getName(); private int splitManagerThreads = Runtime.getRuntime().availableProcessors(); @NotNull @@ -340,6 +338,20 @@ public IcebergConfig setManifestCacheMaxContentLength(long manifestCacheMaxConte return this; } + public String getFileIOContentCacheManager() + { + return fileIOContentCacheManager; + } + + @NotNull + @Config("iceberg.io.manifest.cache.content-caches-impl") + @ConfigDescription("Custom FileIO Content cache manager for file io to use") + public IcebergConfig setFileIOContentCacheManager(String fileIOContentCacheManager) + { + this.fileIOContentCacheManager = fileIOContentCacheManager; + return this; + } + @Min(0) public int getSplitManagerThreads() { diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergFileIOStats.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergFileIOStats.java new file mode 100644 index 0000000000000..e418d6414e583 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergFileIOStats.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import org.weakref.jmx.Managed; + +import javax.inject.Inject; + +public class IcebergFileIOStats +{ + @Inject + private static PrestoInMemoryContentCacheManager inMemoryContentCacheManager; + + private static final String HDFS_FILE_IO = "com.facebook.presto.iceberg.HdfsFileIO"; + private static final String HADOOP_FILE_IO = "org.apache.iceberg.hadoop.HadoopFileIO"; + + @Managed + public String getHiveCatalogIOStats() + { + return inMemoryContentCacheManager.getCache().getIfPresent(HDFS_FILE_IO).stats().toString(); + } + + @Managed + public String getHadoopCatalogIOStats() + { + return inMemoryContentCacheManager.getCache().getIfPresent(HADOOP_FILE_IO).stats().toString(); + } + + @Managed + public String getNessieCatalogIOStats() + { + return inMemoryContentCacheManager.getCache().getIfPresent(HADOOP_FILE_IO).stats().toString(); + } + + @Managed + public String getRestCatalogIOStats() + { + return inMemoryContentCacheManager.getCache().getIfPresent(HADOOP_FILE_IO).stats().toString(); + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java index d7ef841493609..7478e08cd0287 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java @@ -155,6 +155,7 @@ public class IcebergHiveMetadata private final FilterStatsCalculatorService filterStatsCalculatorService; private final IcebergHiveTableOperationsConfig hiveTableOeprationsConfig; + private final IcebergConfig icebergConfig; public IcebergHiveMetadata( ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, @@ -164,13 +165,15 @@ public IcebergHiveMetadata( JsonCodec commitTaskCodec, NodeVersion nodeVersion, FilterStatsCalculatorService filterStatsCalculatorService, - IcebergHiveTableOperationsConfig hiveTableOeprationsConfig) + IcebergHiveTableOperationsConfig hiveTableOeprationsConfig, + IcebergConfig icebergConfig) { super(typeManager, functionResolution, rowExpressionService, commitTaskCodec, nodeVersion); this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); this.hiveTableOeprationsConfig = requireNonNull(hiveTableOeprationsConfig, "hiveTableOperationsConfig is null"); + this.icebergConfig = requireNonNull(icebergConfig, "icebergConfig is null"); } public ExtendedHiveMetastore getMetastore() @@ -181,7 +184,7 @@ public ExtendedHiveMetastore getMetastore() @Override protected org.apache.iceberg.Table getRawIcebergTable(ConnectorSession session, SchemaTableName schemaTableName) { - return getHiveIcebergTable(metastore, hdfsEnvironment, hiveTableOeprationsConfig, session, schemaTableName); + return getHiveIcebergTable(metastore, hdfsEnvironment, hiveTableOeprationsConfig, icebergConfig, session, schemaTableName); } @Override @@ -300,6 +303,7 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con hdfsEnvironment, hdfsContext, hiveTableOeprationsConfig, + icebergConfig, schemaName, tableName, session.getUser(), diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java index f8175f1b0c856..ea357e49c4e04 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java @@ -39,6 +39,7 @@ public class IcebergHiveMetadataFactory final NodeVersion nodeVersion; final FilterStatsCalculatorService filterStatsCalculatorService; final IcebergHiveTableOperationsConfig operationsConfig; + final IcebergConfig icebergConfig; @Inject public IcebergHiveMetadataFactory( @@ -50,7 +51,8 @@ public IcebergHiveMetadataFactory( JsonCodec commitTaskCodec, NodeVersion nodeVersion, FilterStatsCalculatorService filterStatsCalculatorService, - IcebergHiveTableOperationsConfig operationsConfig) + IcebergHiveTableOperationsConfig operationsConfig, + IcebergConfig icebergConfig) { this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); @@ -61,6 +63,7 @@ public IcebergHiveMetadataFactory( this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null"); this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); this.operationsConfig = requireNonNull(operationsConfig, "operationsConfig is null"); + this.icebergConfig = requireNonNull(icebergConfig, "icebergConfig is null"); } public ConnectorMetadata create() @@ -74,6 +77,7 @@ public ConnectorMetadata create() commitTaskCodec, nodeVersion, filterStatsCalculatorService, - operationsConfig); + operationsConfig, + icebergConfig); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index b079c89f14f33..470c75eeb81de 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -168,6 +168,8 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS; import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE; import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP; +import static org.apache.iceberg.CatalogProperties.FILE_IO_IMPL; +import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_CONTENT_CACHE_MANAGER_IMPL; import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_ENABLED; import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS; import static org.apache.iceberg.CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH; @@ -222,7 +224,7 @@ public static Table getIcebergTable(ConnectorMetadata metadata, ConnectorSession return icebergMetadata.getIcebergTable(session, table); } - public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, IcebergHiveTableOperationsConfig config, ConnectorSession session, SchemaTableName table) + public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, IcebergHiveTableOperationsConfig config, IcebergConfig icebergConfig, ConnectorSession session, SchemaTableName table) { HdfsContext hdfsContext = new HdfsContext(session, table.getSchemaName(), table.getTableName()); TableOperations operations = new HiveTableOperations( @@ -231,6 +233,7 @@ public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnv hdfsEnvironment, hdfsContext, config, + icebergConfig, table.getSchemaName(), table.getTableName()); return new BaseTable(operations, quotedTableName(table)); @@ -775,9 +778,11 @@ public static Map getPartitionKeys(PartitionSpec spec public static void loadCachingProperties(Map properties, IcebergConfig icebergConfig) { properties.put(IO_MANIFEST_CACHE_ENABLED, "true"); + properties.put(FILE_IO_IMPL, String.valueOf(icebergConfig.getFileIOImpl())); properties.put(IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, String.valueOf(icebergConfig.getMaxManifestCacheSize())); properties.put(IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, String.valueOf(icebergConfig.getManifestCacheMaxContentLength())); properties.put(IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, String.valueOf(icebergConfig.getManifestCacheExpireDuration())); + properties.put(IO_MANIFEST_CACHE_CONTENT_CACHE_MANAGER_IMPL, String.valueOf(icebergConfig.getFileIOContentCacheManager())); } public static long getDataSequenceNumber(ContentFile file) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PrestoInMemoryContentCacheManager.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PrestoInMemoryContentCacheManager.java new file mode 100644 index 0000000000000..1199e3f51425a --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/PrestoInMemoryContentCacheManager.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.airlift.log.Logger; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.SystemConfigs; +import org.apache.iceberg.io.ContentCacheManager; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.FileIOContentCache; +import org.apache.iceberg.io.InMemoryContentCache; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.util.PropertyUtil; + +import javax.inject.Inject; + +import java.util.Map; + +public final class PrestoInMemoryContentCacheManager + implements ContentCacheManager +{ + private static final Logger LOG = Logger.get(PrestoInMemoryContentCacheManager.class); + + public Cache getCache() + { + return cache; + } + + private final Cache cache = newManifestCacheBuilder().build(); + @Inject + private static PrestoInMemoryContentCacheManager inMemoryContentCacheManager; + + public static PrestoInMemoryContentCacheManager create(Map unused) + { + return inMemoryContentCacheManager; + } + + @Override + public FileIOContentCache contentCache(FileIO io) + { + return cache.get( + io.getClass().getCanonicalName(), + fileIO -> + new InMemoryContentCache( + cacheDurationMs(io), cacheTotalBytes(io), cacheMaxContentLength(io))); + } + + @Override + public synchronized void dropCache(FileIO fileIO) + { + cache.invalidate(fileIO); + cache.cleanUp(); + } + + static long cacheDurationMs(FileIO io) + { + return PropertyUtil.propertyAsLong( + io.properties(), + CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, + CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT); + } + + static long cacheTotalBytes(FileIO io) + { + return PropertyUtil.propertyAsLong( + io.properties(), + CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, + CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT); + } + + static long cacheMaxContentLength(FileIO io) + { + return PropertyUtil.propertyAsLong( + io.properties(), + CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, + CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT); + } + + @VisibleForTesting + static Caffeine newManifestCacheBuilder() + { + int maxSize = SystemConfigs.IO_MANIFEST_CACHE_MAX_FILEIO.value(); + return Caffeine.newBuilder() + .softValues() + .maximumSize(maxSize) + .removalListener( + (io, contentCache, cause) -> + LOG.debug("Evicted {} from FileIO-level cache ({})", io, cause)) + .recordStats(); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java index 7bd7fb34bacee..b2ff1d4779eab 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; +import com.facebook.presto.iceberg.IcebergConfig; import com.facebook.presto.iceberg.IcebergDistributedTestBase; import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; import com.facebook.presto.iceberg.IcebergUtil; @@ -68,6 +69,7 @@ protected Table loadTable(String tableName) return IcebergUtil.getHiveIcebergTable(getFileHiveMetastore(), getHdfsEnvironment(), new IcebergHiveTableOperationsConfig(), + new IcebergConfig(), getQueryRunner().getDefaultSession().toConnectorSession(connectorId), SchemaTableName.valueOf("tpch." + tableName)); } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java index d83095286dfa7..da7c6557b81fc 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java @@ -80,6 +80,7 @@ protected Table getIcebergTable(ConnectorSession session, String schema, String return IcebergUtil.getHiveIcebergTable(getFileHiveMetastore(), getHdfsEnvironment(), new IcebergHiveTableOperationsConfig(), + new IcebergConfig(), session, SchemaTableName.valueOf(schema + "." + tableName)); } diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index ac8f61a3f00a4..eed8edb51e422 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -18,6 +18,13 @@ + + com.google.errorprone + error_prone_annotations + 2.10.0 + runtime + + com.google.guava guava