diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java index 96d4232b6a4d..7e053467e0ac 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastore.java @@ -128,7 +128,10 @@ public static CachingHiveMetastoreBuilder builder(CachingHiveMetastoreBuilder ot return new CachingHiveMetastoreBuilder( other.delegate, other.executor, + other.metadataCacheEnabled, + other.statsCacheEnabled, other.expiresAfterWriteMillis, + other.statsExpiresAfterWriteMillis, other.refreshMills, other.maximumSize, other.statsRecording, @@ -139,6 +142,8 @@ public static CachingHiveMetastore memoizeMetastore(HiveMetastore delegate, long { return builder() .delegate(delegate) + .metadataCacheEnabled(true) + .statsCacheEnabled(true) .maximumSize(maximumSize) .statsRecording(StatsRecording.DISABLED) .build(); @@ -149,7 +154,10 @@ public static class CachingHiveMetastoreBuilder { private HiveMetastore delegate; private Optional executor = Optional.empty(); + private Boolean metadataCacheEnabled; + private Boolean statsCacheEnabled; private OptionalLong expiresAfterWriteMillis = OptionalLong.empty(); + private OptionalLong statsExpiresAfterWriteMillis = OptionalLong.empty(); private OptionalLong refreshMills = OptionalLong.empty(); private Long maximumSize; private StatsRecording statsRecording = StatsRecording.ENABLED; @@ -160,7 +168,10 @@ public CachingHiveMetastoreBuilder() {} private CachingHiveMetastoreBuilder( HiveMetastore delegate, Optional executor, + boolean metadataCacheEnabled, + boolean statsCacheEnabled, OptionalLong expiresAfterWriteMillis, + OptionalLong statsExpiresAfterWriteMillis, OptionalLong refreshMills, Long maximumSize, StatsRecording statsRecording, @@ -168,7 +179,10 @@ private CachingHiveMetastoreBuilder( { this.delegate = delegate; this.executor = executor; + this.metadataCacheEnabled = metadataCacheEnabled; + this.statsCacheEnabled = statsCacheEnabled; this.expiresAfterWriteMillis = expiresAfterWriteMillis; + this.statsExpiresAfterWriteMillis = statsExpiresAfterWriteMillis; this.refreshMills = refreshMills; this.maximumSize = maximumSize; this.statsRecording = statsRecording; @@ -189,6 +203,20 @@ public CachingHiveMetastoreBuilder executor(Executor executor) return this; } + @CanIgnoreReturnValue + public CachingHiveMetastoreBuilder metadataCacheEnabled(boolean metadataCacheEnabled) + { + this.metadataCacheEnabled = metadataCacheEnabled; + return this; + } + + @CanIgnoreReturnValue + public CachingHiveMetastoreBuilder statsCacheEnabled(boolean statsCacheEnabled) + { + this.statsCacheEnabled = statsCacheEnabled; + return this; + } + @CanIgnoreReturnValue public CachingHiveMetastoreBuilder cacheTtl(Duration cacheTtl) { @@ -196,6 +224,13 @@ public CachingHiveMetastoreBuilder cacheTtl(Duration cacheTtl) return this; } + @CanIgnoreReturnValue + public CachingHiveMetastoreBuilder statsCacheTtl(Duration statsCacheTtl) + { + statsExpiresAfterWriteMillis = OptionalLong.of(requireNonNull(statsCacheTtl, "statsCacheTtl is null").toMillis()); + return this; + } + @CanIgnoreReturnValue public CachingHiveMetastoreBuilder refreshInterval(Duration refreshInterval) { @@ -235,11 +270,16 @@ public CachingHiveMetastoreBuilder partitionCacheEnabled(boolean partitionCacheE public CachingHiveMetastore build() { + requireNonNull(metadataCacheEnabled, "metadataCacheEnabled not set"); + requireNonNull(statsCacheEnabled, "statsCacheEnabled is null"); requireNonNull(delegate, "delegate not set"); requireNonNull(maximumSize, "maximumSize not set"); return new CachingHiveMetastore( delegate, + metadataCacheEnabled, + statsCacheEnabled, expiresAfterWriteMillis, + statsExpiresAfterWriteMillis, refreshMills, executor, maximumSize, @@ -248,18 +288,49 @@ public CachingHiveMetastore build() } } - protected CachingHiveMetastore(HiveMetastore delegate, OptionalLong expiresAfterWriteMillis, OptionalLong refreshMills, Optional executor, long maximumSize, StatsRecording statsRecording, boolean partitionCacheEnabled) + protected CachingHiveMetastore( + HiveMetastore delegate, + boolean metadataCacheEnabled, + boolean statsCacheEnabled, + OptionalLong expiresAfterWriteMillis, + OptionalLong statsExpiresAfterWriteMillis, + OptionalLong refreshMills, + Optional executor, + long maximumSize, + StatsRecording statsRecording, + boolean partitionCacheEnabled) { + checkArgument(metadataCacheEnabled || statsCacheEnabled, "Cache not enabled"); this.delegate = requireNonNull(delegate, "delegate is null"); requireNonNull(executor, "executor is null"); - CacheFactory cacheFactory = cacheFactory(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording); - CacheFactory partitionCacheFactory = partitionCacheEnabled ? cacheFactory : neverCacheFactory(); + CacheFactory cacheFactory; + CacheFactory partitionCacheFactory; + if (metadataCacheEnabled) { + cacheFactory = cacheFactory(expiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording); + partitionCacheFactory = partitionCacheEnabled ? cacheFactory : neverCacheFactory(); + } + else { + cacheFactory = neverCacheFactory(); + partitionCacheFactory = neverCacheFactory(); + } + + CacheFactory statsCacheFactory; + CacheFactory partitionStatsCacheFactory; + if (statsCacheEnabled) { + statsCacheFactory = cacheFactory(statsExpiresAfterWriteMillis, refreshMills, executor, maximumSize, statsRecording); + partitionStatsCacheFactory = partitionCacheEnabled ? statsCacheFactory : neverCacheFactory(); + } + else { + statsCacheFactory = neverCacheFactory(); + partitionStatsCacheFactory = neverCacheFactory(); + } + databaseNamesCache = cacheFactory.buildCache(ignored -> loadAllDatabases()); databaseCache = cacheFactory.buildCache(this::loadDatabase); tableNamesCache = cacheFactory.buildCache(this::loadAllTables); tablesWithParameterCache = cacheFactory.buildCache(this::loadTablesMatchingParameter); - tableStatisticsCache = cacheFactory.buildCache(this::loadTableColumnStatistics); + tableStatisticsCache = statsCacheFactory.buildCache(this::loadTableColumnStatistics); tableCache = cacheFactory.buildCache(this::loadTable); viewNamesCache = cacheFactory.buildCache(this::loadAllViews); tablePrivilegesCache = cacheFactory.buildCache(key -> loadTablePrivileges(key.getDatabase(), key.getTable(), key.getOwner(), key.getPrincipal())); @@ -268,7 +339,7 @@ protected CachingHiveMetastore(HiveMetastore delegate, OptionalLong expiresAfter grantedPrincipalsCache = cacheFactory.buildCache(this::loadPrincipals); configValuesCache = cacheFactory.buildCache(this::loadConfigValue); - partitionStatisticsCache = partitionCacheFactory.buildBulkCache(); + partitionStatisticsCache = partitionStatsCacheFactory.buildBulkCache(); partitionFilterCache = partitionCacheFactory.buildCache(this::loadPartitionNamesByFilter); partitionCache = partitionCacheFactory.buildBulkCache(); } @@ -739,7 +810,8 @@ public Optional getPartition(Table table, List partitionValue } @Override - public Optional> getPartitionNamesByFilter(String databaseName, + public Optional> getPartitionNamesByFilter( + String databaseName, String tableName, List columnNames, TupleDomain partitionKeysFilter) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java index 38b38803d941..e5b53df08c38 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/CachingHiveMetastoreConfig.java @@ -26,6 +26,11 @@ public class CachingHiveMetastoreConfig { private Duration metastoreCacheTtl = new Duration(0, TimeUnit.SECONDS); + // Use 5 mins for stats cache TTL by default. 5 mins will be sufficient to help + // significantly when there is high number of concurrent queries. + // 5 mins will also prevent stats from being stalled for a long time since + // time window where table data can be altered is limited. + private Duration statsCacheTtl = new Duration(5, TimeUnit.MINUTES); private Optional metastoreRefreshInterval = Optional.empty(); private long metastoreCacheMaximumSize = 10000; private int maxMetastoreRefreshThreads = 10; @@ -44,6 +49,19 @@ public CachingHiveMetastoreConfig setMetastoreCacheTtl(Duration metastoreCacheTt return this; } + @NotNull + public Duration getStatsCacheTtl() + { + return statsCacheTtl; + } + + @Config("hive.metastore-stats-cache-ttl") + public CachingHiveMetastoreConfig setStatsCacheTtl(Duration statsCacheTtl) + { + this.statsCacheTtl = statsCacheTtl; + return this; + } + @NotNull public Optional<@MinDuration("1ms") Duration> getMetastoreRefreshInterval() { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java index f263cbdf005a..a1994e447726 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/cache/SharedHiveMetastoreCache.java @@ -78,12 +78,17 @@ public SharedHiveMetastoreCache( // Disable caching on workers, because there currently is no way to invalidate such a cache. // Note: while we could skip CachingHiveMetastoreModule altogether on workers, we retain it so that catalog // configuration can remain identical for all nodes, making cluster configuration easier. - enabled = nodeManager.getCurrentNode().isCoordinator() && - config.getMetastoreCacheTtl().toMillis() > 0 && + boolean metadataCacheEnabled = config.getMetastoreCacheTtl().toMillis() > 0; + boolean statsCacheEnabled = config.getStatsCacheTtl().toMillis() > 0; + enabled = (metadataCacheEnabled || statsCacheEnabled) && + nodeManager.getCurrentNode().isCoordinator() && config.getMetastoreCacheMaximumSize() > 0; cachingMetastoreBuilder = CachingHiveMetastore.builder() + .metadataCacheEnabled(metadataCacheEnabled) + .statsCacheEnabled(statsCacheEnabled) .cacheTtl(config.getMetastoreCacheTtl()) + .statsCacheTtl(config.getStatsCacheTtl()) .refreshInterval(config.getMetastoreRefreshInterval()) .maximumSize(config.getMetastoreCacheMaximumSize()) .partitionCacheEnabled(config.isPartitionCacheEnabled()); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index cc4f72c19f16..e958d591edfe 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -796,6 +796,8 @@ protected final void setup(HostAndPort metastoreAddress, String databaseName) .hdfsEnvironment(hdfsEnvironment) .build())) .executor(executor) + .metadataCacheEnabled(true) + .statsCacheEnabled(true) .cacheTtl(new Duration(1, MINUTES)) .refreshInterval(new Duration(15, SECONDS)) .maximumSize(10000) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java index 4e43ec4b4931..d48771acede5 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastore.java @@ -113,6 +113,7 @@ public class TestCachingHiveMetastore private MockThriftMetastoreClient mockClient; private ListeningExecutorService executor; private CachingHiveMetastore metastore; + private CachingHiveMetastore statsCacheMetastore; private ThriftMetastoreStats stats; @BeforeMethod @@ -124,6 +125,18 @@ public void setUp() metastore = CachingHiveMetastore.builder() .delegate(new BridgingHiveMetastore(thriftHiveMetastore)) .executor(executor) + .metadataCacheEnabled(true) + .statsCacheEnabled(true) + .cacheTtl(new Duration(5, TimeUnit.MINUTES)) + .refreshInterval(new Duration(1, TimeUnit.MINUTES)) + .maximumSize(1000) + .partitionCacheEnabled(true) + .build(); + statsCacheMetastore = CachingHiveMetastore.builder() + .delegate(new BridgingHiveMetastore(thriftHiveMetastore)) + .executor(executor) + .metadataCacheEnabled(false) + .statsCacheEnabled(true) // only cache stats .cacheTtl(new Duration(5, TimeUnit.MINUTES)) .refreshInterval(new Duration(1, TimeUnit.MINUTES)) .maximumSize(1000) @@ -502,6 +515,27 @@ public void testGetTableStatistics() assertEquals(metastore.getTableStats().getHitRate(), 0.0); } + @Test + public void testGetTableStatisticsWithoutMetadataCache() + { + assertEquals(mockClient.getAccessCount(), 0); + + Table table = statsCacheMetastore.getTable(TEST_DATABASE, TEST_TABLE).orElseThrow(); + assertEquals(mockClient.getAccessCount(), 1); + + assertEquals(statsCacheMetastore.getTableStatistics(table), TEST_STATS); + assertEquals(mockClient.getAccessCount(), 2); + + assertEquals(statsCacheMetastore.getTableStatistics(table), TEST_STATS); + assertEquals(mockClient.getAccessCount(), 2); + + assertEquals(statsCacheMetastore.getTableStatisticsStats().getRequestCount(), 2); + assertEquals(statsCacheMetastore.getTableStatisticsStats().getHitRate(), 0.5); + + assertEquals(statsCacheMetastore.getTableStats().getRequestCount(), 0); + assertEquals(statsCacheMetastore.getTableStats().getHitRate(), 1.0); + } + @Test public void testGetPartitionStatistics() { @@ -529,6 +563,33 @@ public void testGetPartitionStatistics() assertEquals(metastore.getPartitionStats().getHitRate(), 0.0); } + @Test + public void testGetPartitionStatisticsWithoutMetadataCache() + { + assertEquals(mockClient.getAccessCount(), 0); + + Table table = statsCacheMetastore.getTable(TEST_DATABASE, TEST_TABLE).orElseThrow(); + assertEquals(mockClient.getAccessCount(), 1); + + Partition partition = statsCacheMetastore.getPartition(table, TEST_PARTITION_VALUES1).orElseThrow(); + assertEquals(mockClient.getAccessCount(), 2); + + assertEquals(statsCacheMetastore.getPartitionStatistics(table, ImmutableList.of(partition)), ImmutableMap.of(TEST_PARTITION1, TEST_STATS)); + assertEquals(mockClient.getAccessCount(), 3); + + assertEquals(statsCacheMetastore.getPartitionStatistics(table, ImmutableList.of(partition)), ImmutableMap.of(TEST_PARTITION1, TEST_STATS)); + assertEquals(mockClient.getAccessCount(), 3); + + assertEquals(statsCacheMetastore.getPartitionStatisticsStats().getRequestCount(), 2); + assertEquals(statsCacheMetastore.getPartitionStatisticsStats().getHitRate(), 0.5); + + assertEquals(statsCacheMetastore.getTableStats().getRequestCount(), 0); + assertEquals(statsCacheMetastore.getTableStats().getHitRate(), 1.0); + + assertEquals(statsCacheMetastore.getPartitionStats().getRequestCount(), 0); + assertEquals(statsCacheMetastore.getPartitionStats().getHitRate(), 1.0); + } + @Test public void testUpdatePartitionStatistics() { @@ -683,6 +744,8 @@ public Map> getPartitionsByNames(Table table, List properties = ImmutableMap.builder() .put("hive.metastore-cache-ttl", "2h") + .put("hive.metastore-stats-cache-ttl", "10m") .put("hive.metastore-refresh-interval", "30m") .put("hive.metastore-cache-maximum-size", "5000") .put("hive.metastore-refresh-max-threads", "2500") @@ -50,6 +52,7 @@ public void testExplicitPropertyMappings() CachingHiveMetastoreConfig expected = new CachingHiveMetastoreConfig() .setMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS)) + .setStatsCacheTtl(new Duration(10, TimeUnit.MINUTES)) .setMetastoreRefreshInterval(new Duration(30, TimeUnit.MINUTES)) .setMetastoreCacheMaximumSize(5000) .setMaxMetastoreRefreshThreads(2500) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java index 0fbd280a241d..bd865e4190fa 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/file/IcebergFileMetastoreCatalogModule.java @@ -17,14 +17,19 @@ import com.google.inject.Key; import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.airlift.units.Duration; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.DecoratedHiveMetastoreModule; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.file.FileMetastoreModule; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.MetastoreValidator; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalogFactory; +import java.util.concurrent.TimeUnit; + +import static io.airlift.configuration.ConfigBinder.configBinder; import static io.trino.plugin.iceberg.catalog.hms.IcebergHiveMetastoreCatalogModule.HIDE_DELTA_LAKE_TABLES_IN_ICEBERG; public class IcebergFileMetastoreCatalogModule @@ -39,5 +44,10 @@ protected void setup(Binder binder) binder.bind(MetastoreValidator.class).asEagerSingleton(); binder.bind(Key.get(boolean.class, HideDeltaLakeTables.class)).toInstance(HIDE_DELTA_LAKE_TABLES_IN_ICEBERG); install(new DecoratedHiveMetastoreModule()); + + configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { + // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg + config.setStatsCacheTtl(new Duration(0, TimeUnit.SECONDS)); + }); } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java index f6838f1f5083..3dced4066341 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/IcebergHiveMetastoreCatalogModule.java @@ -17,14 +17,20 @@ import com.google.inject.Key; import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.airlift.units.Duration; import io.trino.plugin.hive.HideDeltaLakeTables; import io.trino.plugin.hive.metastore.DecoratedHiveMetastoreModule; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreModule; import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.MetastoreValidator; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; +import java.util.concurrent.TimeUnit; + +import static io.airlift.configuration.ConfigBinder.configBinder; + public class IcebergHiveMetastoreCatalogModule extends AbstractConfigurationAwareModule { @@ -40,5 +46,10 @@ protected void setup(Binder binder) binder.bind(Key.get(boolean.class, TranslateHiveViews.class)).toInstance(false); binder.bind(Key.get(boolean.class, HideDeltaLakeTables.class)).toInstance(HIDE_DELTA_LAKE_TABLES_IN_ICEBERG); install(new DecoratedHiveMetastoreModule()); + + configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { + // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg + config.setStatsCacheTtl(new Duration(0, TimeUnit.SECONDS)); + }); } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java index 728d77243269..e83e43b05eb4 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java @@ -16,14 +16,19 @@ import com.google.inject.Binder; import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.airlift.units.Duration; import io.trino.plugin.hive.metastore.DecoratedHiveMetastoreModule; import io.trino.plugin.hive.metastore.HiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalogFactory; +import java.util.concurrent.TimeUnit; + +import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class TestingIcebergFileMetastoreCatalogModule @@ -43,5 +48,10 @@ protected void setup(Binder binder) install(new DecoratedHiveMetastoreModule()); binder.bind(IcebergTableOperationsProvider.class).to(FileMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); + + configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { + // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg + config.setStatsCacheTtl(new Duration(0, TimeUnit.SECONDS)); + }); } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java index 38b932d12a21..0b7e1694c824 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestingIcebergHiveMetastoreCatalogModule.java @@ -16,14 +16,19 @@ import com.google.inject.Binder; import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.airlift.units.Duration; import io.trino.plugin.hive.metastore.DecoratedHiveMetastoreModule; import io.trino.plugin.hive.metastore.HiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastoreConfig; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreFactory; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; +import java.util.concurrent.TimeUnit; + +import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class TestingIcebergHiveMetastoreCatalogModule @@ -46,5 +51,10 @@ protected void setup(Binder binder) binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(this.hiveMetastore)); binder.bind(IcebergTableOperationsProvider.class).to(HiveMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); + + configBinder(binder).bindConfigDefaults(CachingHiveMetastoreConfig.class, config -> { + // ensure caching metastore wrapper isn't created, as it's not leveraged by Iceberg + config.setStatsCacheTtl(new Duration(0, TimeUnit.SECONDS)); + }); } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestExternalHiveTable.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestExternalHiveTable.java index ffe781263218..1d1e31ea540e 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestExternalHiveTable.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestExternalHiveTable.java @@ -71,6 +71,7 @@ public void testShowStatisticsForExternalTable() row(null, null, null, null, 5.0, null, null)); onHive().executeQuery("ANALYZE TABLE " + EXTERNAL_TABLE_NAME + " PARTITION (p_regionkey) COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery("SHOW STATS FOR " + EXTERNAL_TABLE_NAME)).containsOnly( row("p_nationkey", null, 5.0, 0.0, null, "1", "24"), row("p_name", 38.0, 5.0, 0.0, null, null, null), diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java index 19f8484bcfb5..d2faf541c4ee 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveTableStatistics.java @@ -200,6 +200,7 @@ public void testStatisticsForUnpartitionedTable() // basic analysis onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("n_nationkey", null, null, null, null, null, null), @@ -211,6 +212,7 @@ public void testStatisticsForUnpartitionedTable() // column analysis onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("n_nationkey", null, anyOf(19., 25.), 0.0, null, "0", "24"), @@ -249,6 +251,7 @@ public void testStatisticsForTablePartitionedByBigint() // basic analysis for single partition onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey = \"1\") COMPUTE STATISTICS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, null, null, null, null, null), @@ -274,6 +277,7 @@ public void testStatisticsForTablePartitionedByBigint() // basic analysis for all partitions onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey) COMPUTE STATISTICS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, null, null, null, null, null), @@ -299,6 +303,7 @@ public void testStatisticsForTablePartitionedByBigint() // column analysis for single partition onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey = \"1\") COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, 5.0, 0.0, null, "1", "24"), @@ -324,6 +329,7 @@ public void testStatisticsForTablePartitionedByBigint() // column analysis for all partitions onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey) COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, 5.0, 0.0, null, "1", "24"), @@ -376,6 +382,7 @@ public void testStatisticsForTablePartitionedByVarchar() // basic analysis for single partition onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey = \"AMERICA\") COMPUTE STATISTICS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, null, null, null, null, null), @@ -401,6 +408,7 @@ public void testStatisticsForTablePartitionedByVarchar() // basic analysis for all partitions onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey) COMPUTE STATISTICS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, null, null, null, null, null), @@ -426,6 +434,7 @@ public void testStatisticsForTablePartitionedByVarchar() // column analysis for single partition onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey = \"AMERICA\") COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, 5.0, 0.0, null, "1", "24"), @@ -451,6 +460,7 @@ public void testStatisticsForTablePartitionedByVarchar() // column analysis for all partitions onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " PARTITION (p_regionkey) COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery(showStatsWholeTable)).containsOnly( row("p_nationkey", null, 5.0, 0.0, null, "1", "24"), @@ -502,6 +512,7 @@ public void testStatisticsForAllDataTypes() row(null, null, null, null, 2.0, null, null)); onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); // SHOW STATS FORMAT: column_name, data_size, distinct_values_count, nulls_fraction, row_count assertThat(onTrino().executeQuery("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( @@ -550,6 +561,7 @@ public void testStatisticsForAllDataTypesNoData() row(null, null, null, null, 0.0, null, null)); onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( row("c_tinyint", 0.0, 0.0, 1.0, null, null, null), @@ -598,6 +610,7 @@ public void testStatisticsForAllDataTypesOnlyNulls() row(null, null, null, null, 1.0, null, null)); onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( row("c_tinyint", 0.0, 0.0, 1.0, null, null, null), @@ -633,6 +646,7 @@ public void testStatisticsForSkewedTable() row(null, null, null, null, 2.0, null, null)); onHive().executeQuery("ANALYZE TABLE " + tableName + " COMPUTE STATISTICS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery("SHOW STATS FOR " + tableName)).containsOnly( row("c_string", null, null, null, null, null, null), @@ -640,6 +654,7 @@ public void testStatisticsForSkewedTable() row(null, null, null, null, 2.0, null, null)); onHive().executeQuery("ANALYZE TABLE " + tableName + " COMPUTE STATISTICS FOR COLUMNS"); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); assertThat(onTrino().executeQuery("SHOW STATS FOR " + tableName)).containsOnly( row("c_string", 4.0, 1.0, 0.0, null, null, null), row("c_int", null, 2.0, 0.0, null, "1", "2"), @@ -1413,6 +1428,7 @@ public void testMixedHiveAndPrestoStatistics() onTrino().executeQuery(format("ANALYZE %s WITH (partitions = ARRAY[ARRAY['1']])", tableName)); onHive().executeQuery(format("ANALYZE TABLE %s PARTITION (p = \"2\") COMPUTE STATISTICS", tableName)); onHive().executeQuery(format("ANALYZE TABLE %s PARTITION (p = \"2\") COMPUTE STATISTICS FOR COLUMNS", tableName)); + onTrino().executeQuery("CALL system.flush_metadata_cache()"); // we can get stats for individual partitions assertThat(onTrino().executeQuery(showStatsPartitionOne)).containsOnly(