From d882b49323d528df7342f666d4f10c1c4018f4e6 Mon Sep 17 00:00:00 2001 From: Zac Blanco Date: Wed, 10 Jul 2024 14:36:17 -0700 Subject: [PATCH] [Iceberg] Add table refresh retry configurations --- .../src/main/sphinx/connector/iceberg.rst | 98 ++++++++++------ .../presto/iceberg/HiveTableOperations.java | 29 +++-- .../presto/iceberg/IcebergHiveMetadata.java | 9 +- .../iceberg/IcebergHiveMetadataFactory.java | 16 ++- .../presto/iceberg/IcebergHiveModule.java | 1 + .../IcebergHiveTableOperationsConfig.java | 105 ++++++++++++++++++ .../facebook/presto/iceberg/IcebergUtil.java | 3 +- .../iceberg/IcebergDistributedTestBase.java | 1 - .../iceberg/TestIcebergMetadataListing.java | 3 +- .../hive/TestHiveTableOperationsConfig.java | 63 +++++++++++ .../hive/TestIcebergDistributedHive.java | 2 + .../iceberg/hive/TestIcebergSmokeHive.java | 2 + 12 files changed, 282 insertions(+), 50 deletions(-) create mode 100644 presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java create mode 100644 presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestHiveTableOperationsConfig.java diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index d33f73d0a5936..bba13162da08f 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -12,8 +12,8 @@ Metastores Iceberg tables store most of the metadata in the metadata files, along with the data on the filesystem, but it still requires a central place to find the current location of the current metadata pointer for a table. This central place is called the ``Iceberg Catalog``. -The Presto Iceberg connector supports different types of Iceberg Catalogs : ``Hive Metastore``, -``GLUE``, ``NESSIE``, ``REST`` and ``HADOOP``. +The Presto Iceberg connector supports different types of Iceberg Catalogs : ``HIVE``, +``NESSIE``, ``REST``, and ``HADOOP``. To configure the Iceberg connector, create a catalog properties file ``etc/catalog/iceberg.properties``. To define the catalog type, ``iceberg.catalog.type`` property @@ -45,6 +45,48 @@ as a Hive connector. hive.metastore=glue iceberg.catalog.type=hive +There are additional configurations available when using the Iceberg connector configured with Hive +or Glue catalogs. + +======================================================== ============================================================= ============ +Property Name Description Default +======================================================== ============================================================= ============ +``hive.metastore.uri`` The URI(s) of the Hive metastore to connect to using the + Thrift protocol. If multiple URIs are provided, the first + URI is used by default, and the rest of the URIs are + fallback metastores. + + Example: ``thrift://192.0.2.3:9083`` or + ``thrift://192.0.2.3:9083,thrift://192.0.2.4:9083``. + + This property is required if the + ``iceberg.catalog.type`` is ``hive`` and ``hive.metastore`` + is ``thrift``. + +``iceberg.hive-statistics-merge-strategy`` Comma separated list of statistics to use from the + Hive Metastore to override Iceberg table statistics. + The available values are ``NUMBER_OF_DISTINCT_VALUES`` + and ``TOTAL_SIZE_IN_BYTES``. + + **Note**: Only valid when the Iceberg connector is + configured with Hive. + +``iceberg.hive.table-refresh.backoff-min-sleep-time`` The minimum amount of time to sleep between retries when 100ms + refreshing table metadata. + +``iceberg.hive.table-refresh.backoff-max-sleep-time`` The maximum amount of time to sleep between retries when 5s + refreshing table metadata. + +``iceberg.hive.table-refresh.max-retry-time`` The maximum amount of time to take across all retries before 1min + failing a table metadata refresh operation. + +``iceberg.hive.table-refresh.retries`` The number of times to retry after errors when refreshing 20 + table metadata using the Hive metastore. + +``iceberg.hive.table-refresh.backoff-scale-factor`` The multiple used to scale subsequent wait time between 4.0 + retries. +======================================================== ============================================================= ============ + Nessie catalog ^^^^^^^^^^^^^^ @@ -194,35 +236,11 @@ To use a Hadoop catalog, configure the catalog type as iceberg.catalog.type=hadoop iceberg.catalog.warehouse=hdfs://hostname:port -Configuration Properties ------------------------- - -.. note:: - - The Iceberg connector supports configuration options for - `Amazon S3 `_ - as a Hive connector. - -The following configuration properties are available: +Hadoop catalog configuration properties: ======================================================= ============================================================= ============ Property Name Description Default ======================================================= ============================================================= ============ -``hive.metastore.uri`` The URI(s) of the Hive metastore to connect to using the - Thrift protocol. If multiple URIs are provided, the first - URI is used by default, and the rest of the URIs are - fallback metastores. - - Example: ``thrift://192.0.2.3:9083`` or - ``thrift://192.0.2.3:9083,thrift://192.0.2.4:9083``. - - This property is required if the - ``iceberg.catalog.type`` is ``hive``. Otherwise, it will - be ignored. - -``iceberg.catalog.type`` The catalog type for Iceberg tables. The available values ``hive`` - are ``hive``, ``hadoop``, and ``nessie``. - ``iceberg.catalog.warehouse`` The catalog warehouse root path for Iceberg tables. Example: ``hdfs://nn:8020/warehouse/path`` @@ -232,6 +250,24 @@ Property Name Description ``iceberg.catalog.cached-catalog-num`` The number of Iceberg catalogs to cache. This property is ``10`` required if the ``iceberg.catalog.type`` is ``hadoop``. Otherwise, it will be ignored. +======================================================= ============================================================= ============ + +Configuration Properties +------------------------ + +.. note:: + + The Iceberg connector supports configuration options for + `Amazon S3 `_ + as a Hive connector. + +The following configuration properties are available for all catalog types: + +======================================================= ============================================================= ============ +Property Name Description Default +======================================================= ============================================================= ============ +``iceberg.catalog.type`` The catalog type for Iceberg tables. The available values ``HIVE`` + are ``HIVE``, ``HADOOP``, and ``NESSIE`` and ``REST``. ``iceberg.hadoop.config.resources`` The path(s) for Hadoop configuration resources. @@ -262,14 +298,6 @@ Property Name Description ``iceberg.enable-parquet-dereference-pushdown`` Enable parquet dereference pushdown. ``true`` -``iceberg.hive-statistics-merge-strategy`` Comma separated list of statistics to use from the - Hive Metastore to override Iceberg table statistics. - The available values are ``NUMBER_OF_DISTINCT_VALUES`` - and ``TOTAL_SIZE_IN_BYTES``. - - **Note**: Only valid when the Iceberg connector is - configured with Hive. - ``iceberg.statistic-snapshot-record-difference-weight`` The amount that the difference in total record count matters when calculating the closest snapshot when picking statistics. A value of 1 means a single record is equivalent diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java index cb3b1a467ce9c..0eff5fdb4a607 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java @@ -101,6 +101,7 @@ public class HiveTableOperations private final Optional owner; private final Optional location; private final FileIO fileIO; + private final IcebergHiveTableOperationsConfig config; private TableMetadata currentMetadata; private String currentMetadataLocation; @@ -114,12 +115,14 @@ public HiveTableOperations( MetastoreContext metastoreContext, HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, + IcebergHiveTableOperationsConfig config, String database, String table) { this(new HdfsFileIO(hdfsEnvironment, hdfsContext), metastore, metastoreContext, + config, database, table, Optional.empty(), @@ -131,6 +134,7 @@ public HiveTableOperations( MetastoreContext metastoreContext, HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, + IcebergHiveTableOperationsConfig config, String database, String table, String owner, @@ -139,6 +143,7 @@ public HiveTableOperations( this(new HdfsFileIO(hdfsEnvironment, hdfsContext), metastore, metastoreContext, + config, database, table, Optional.of(requireNonNull(owner, "owner is null")), @@ -149,6 +154,7 @@ private HiveTableOperations( FileIO fileIO, ExtendedHiveMetastore metastore, MetastoreContext metastoreContext, + IcebergHiveTableOperationsConfig config, String database, String table, Optional owner, @@ -161,6 +167,7 @@ private HiveTableOperations( this.tableName = requireNonNull(table, "table is null"); this.owner = requireNonNull(owner, "owner is null"); this.location = requireNonNull(location, "location is null"); + this.config = requireNonNull(config, "config is null"); //TODO: duration from config initTableLevelLockCache(TimeUnit.MINUTES.toMillis(10)); } @@ -385,15 +392,23 @@ private void refreshFromMetadataLocation(String newLocation) } AtomicReference newMetadata = new AtomicReference<>(); - Tasks.foreach(newLocation) - .retry(20) - .exponentialBackoff(100, 5000, 600000, 4.0) - .suppressFailureWhenFinished() - .run(metadataLocation -> newMetadata.set( - TableMetadataParser.read(fileIO, io().newInputFile(metadataLocation)))); + try { + Tasks.foreach(newLocation) + .retry(config.getTableRefreshRetries()) + .exponentialBackoff( + config.getTableRefreshBackoffMinSleepTime().toMillis(), + config.getTableRefreshBackoffMaxSleepTime().toMillis(), + config.getTableRefreshMaxRetryTime().toMillis(), + config.getTableRefreshBackoffScaleFactor()) + .run(metadataLocation -> newMetadata.set( + TableMetadataParser.read(fileIO, io().newInputFile(metadataLocation)))); + } + catch (RuntimeException e) { + throw new TableNotFoundException(getSchemaTableName(), "Table metadata is missing", e); + } if (newMetadata.get() == null) { - throw new TableNotFoundException(getSchemaTableName(), "Table metadata is missing."); + throw new TableNotFoundException(getSchemaTableName(), "failed to retrieve table metadata from " + newLocation); } String newUUID = newMetadata.get().uuid(); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java index 26652ab35128b..211a7499427ed 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java @@ -151,8 +151,8 @@ public class IcebergHiveMetadata private final ExtendedHiveMetastore metastore; private final HdfsEnvironment hdfsEnvironment; private final DateTimeZone timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of(TimeZone.getDefault().getID()))); - private final FilterStatsCalculatorService filterStatsCalculatorService; + private final IcebergHiveTableOperationsConfig hiveTableOeprationsConfig; public IcebergHiveMetadata( ExtendedHiveMetastore metastore, @@ -162,12 +162,14 @@ public IcebergHiveMetadata( RowExpressionService rowExpressionService, JsonCodec commitTaskCodec, NodeVersion nodeVersion, - FilterStatsCalculatorService filterStatsCalculatorService) + FilterStatsCalculatorService filterStatsCalculatorService, + IcebergHiveTableOperationsConfig hiveTableOeprationsConfig) { super(typeManager, functionResolution, rowExpressionService, commitTaskCodec, nodeVersion); this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); + this.hiveTableOeprationsConfig = requireNonNull(hiveTableOeprationsConfig, "hiveTableOperationsConfig is null"); } public ExtendedHiveMetastore getMetastore() @@ -178,7 +180,7 @@ public ExtendedHiveMetastore getMetastore() @Override protected org.apache.iceberg.Table getRawIcebergTable(ConnectorSession session, SchemaTableName schemaTableName) { - return getHiveIcebergTable(metastore, hdfsEnvironment, session, schemaTableName); + return getHiveIcebergTable(metastore, hdfsEnvironment, hiveTableOeprationsConfig, session, schemaTableName); } @Override @@ -296,6 +298,7 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con getMetastoreContext(session), hdfsEnvironment, hdfsContext, + hiveTableOeprationsConfig, schemaName, tableName, session.getUser(), diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java index d69e9447772fd..f8175f1b0c856 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java @@ -38,6 +38,7 @@ public class IcebergHiveMetadataFactory final RowExpressionService rowExpressionService; final NodeVersion nodeVersion; final FilterStatsCalculatorService filterStatsCalculatorService; + final IcebergHiveTableOperationsConfig operationsConfig; @Inject public IcebergHiveMetadataFactory( @@ -48,7 +49,8 @@ public IcebergHiveMetadataFactory( RowExpressionService rowExpressionService, JsonCodec commitTaskCodec, NodeVersion nodeVersion, - FilterStatsCalculatorService filterStatsCalculatorService) + FilterStatsCalculatorService filterStatsCalculatorService, + IcebergHiveTableOperationsConfig operationsConfig) { this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); @@ -58,10 +60,20 @@ public IcebergHiveMetadataFactory( this.commitTaskCodec = requireNonNull(commitTaskCodec, "commitTaskCodec is null"); this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null"); this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); + this.operationsConfig = requireNonNull(operationsConfig, "operationsConfig is null"); } public ConnectorMetadata create() { - return new IcebergHiveMetadata(metastore, hdfsEnvironment, typeManager, functionResolution, rowExpressionService, commitTaskCodec, nodeVersion, filterStatsCalculatorService); + return new IcebergHiveMetadata( + metastore, + hdfsEnvironment, + typeManager, + functionResolution, + rowExpressionService, + commitTaskCodec, + nodeVersion, + filterStatsCalculatorService, + operationsConfig); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java index 0168ec5cc3d89..522c313e88451 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java @@ -49,6 +49,7 @@ public void setup(Binder binder) { install(new HiveMetastoreModule(this.connectorId, this.metastore)); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(IcebergHiveTableOperationsConfig.class); configBinder(binder).bindConfig(MetastoreClientConfig.class); binder.bind(PartitionMutator.class).to(HivePartitionMutator.class).in(Scopes.SINGLETON); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java new file mode 100644 index 0000000000000..b04d677ce456c --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveTableOperationsConfig.java @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.Min; + +import static io.airlift.units.Duration.succinctDuration; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class IcebergHiveTableOperationsConfig +{ + private Duration tableRefreshBackoffMinSleepTime = succinctDuration(100, MILLISECONDS); + private Duration tableRefreshBackoffMaxSleepTime = succinctDuration(5, SECONDS); + private Duration tableRefreshMaxRetryTime = succinctDuration(1, MINUTES); + private double tableRefreshBackoffScaleFactor = 4.0; + private int tableRefreshRetries = 20; + + @MinDuration("1ms") + public Duration getTableRefreshBackoffMinSleepTime() + { + return tableRefreshBackoffMinSleepTime; + } + + @Config("iceberg.hive.table-refresh.backoff-min-sleep-time") + @ConfigDescription("The minimum amount of time to sleep between retries when refreshing table metadata") + public IcebergHiveTableOperationsConfig setTableRefreshBackoffMinSleepTime(Duration tableRefreshBackoffMinSleepTime) + { + this.tableRefreshBackoffMinSleepTime = tableRefreshBackoffMinSleepTime; + return this; + } + + @MinDuration("1ms") + public Duration getTableRefreshBackoffMaxSleepTime() + { + return tableRefreshBackoffMaxSleepTime; + } + + @Config("iceberg.hive.table-refresh.backoff-max-sleep-time") + @ConfigDescription("The maximum amount of time to sleep between retries when refreshing table metadata") + public IcebergHiveTableOperationsConfig setTableRefreshBackoffMaxSleepTime(Duration tableRefreshBackoffMaxSleepTime) + { + this.tableRefreshBackoffMaxSleepTime = tableRefreshBackoffMaxSleepTime; + return this; + } + + @MinDuration("1ms") + public Duration getTableRefreshMaxRetryTime() + { + return tableRefreshMaxRetryTime; + } + + @Config("iceberg.hive.table-refresh.max-retry-time") + @ConfigDescription("The maximum amount of time to take across all retries before failing a table metadata refresh operation") + public IcebergHiveTableOperationsConfig setTableRefreshMaxRetryTime(Duration tableRefreshMaxRetryTime) + { + this.tableRefreshMaxRetryTime = tableRefreshMaxRetryTime; + return this; + } + + @Min(1) + public double getTableRefreshBackoffScaleFactor() + { + return tableRefreshBackoffScaleFactor; + } + + @Config("iceberg.hive.table-refresh.backoff-scale-factor") + @ConfigDescription("The multiple used to scale subsequent wait time between retries") + public IcebergHiveTableOperationsConfig setTableRefreshBackoffScaleFactor(double tableRefreshBackoffScaleFactor) + { + this.tableRefreshBackoffScaleFactor = tableRefreshBackoffScaleFactor; + return this; + } + + @Config("iceberg.hive.table-refresh.retries") + @ConfigDescription("The number of times to retry after errors when refreshing table metadata using the Hive metastore") + public IcebergHiveTableOperationsConfig setTableRefreshRetries(int tableRefreshRetries) + { + this.tableRefreshRetries = tableRefreshRetries; + return this; + } + + @Min(0) + public int getTableRefreshRetries() + { + return tableRefreshRetries; + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index 7ccd43b93dd5f..c3fbb76e86191 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -216,7 +216,7 @@ public static Table getIcebergTable(ConnectorMetadata metadata, ConnectorSession return icebergMetadata.getIcebergTable(session, table); } - public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, ConnectorSession session, SchemaTableName table) + public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, IcebergHiveTableOperationsConfig config, ConnectorSession session, SchemaTableName table) { HdfsContext hdfsContext = new HdfsContext(session, table.getSchemaName(), table.getTableName()); TableOperations operations = new HiveTableOperations( @@ -224,6 +224,7 @@ public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnv new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getClientTags(), session.getSource(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER, session.getWarningCollector(), session.getRuntimeStats()), hdfsEnvironment, hdfsContext, + config, table.getSchemaName(), table.getTableName()); return new BaseTable(operations, quotedTableName(table)); diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java index 8fb1d6dedbb1a..5160a9e698709 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java @@ -941,7 +941,6 @@ public void testNDVsAtSnapshot() // assert either case as we don't have good control over the timing of when statistics files are written ColumnStatistics col0Stats = columnStatsFor(statistics, "col0"); ColumnStatistics col1Stats = columnStatsFor(statistics, "col1"); - System.out.printf("distinct @ %s count col0: %s%n", snaps.get(i), col0Stats.getDistinctValuesCount()); final int idx = i; assertEither( () -> assertEquals(col0Stats.getDistinctValuesCount(), Estimate.of(idx)), diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMetadataListing.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMetadataListing.java index 9c0fe46f4e21a..580396a2c9cd9 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMetadataListing.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMetadataListing.java @@ -61,6 +61,7 @@ protected QueryRunner createQueryRunner() Map icebergProperties = ImmutableMap.builder() .put("hive.metastore", "file") .put("hive.metastore.catalog.dir", catalogDirectory.toFile().toURI().toString()) + .put("iceberg.hive.table-refresh.max-retry-time", "500ms") // improves test time for testTableDropWithMissingMetadata .build(); queryRunner.createCatalog(ICEBERG_CATALOG, "iceberg", icebergProperties); @@ -135,7 +136,7 @@ public void testTableDropWithMissingMetadata() } tableMetadataDir.delete(); - assertQueryFails("SELECT * FROM iceberg.test_metadata_schema.iceberg_table1", "Could not read table schema"); + assertQueryFails("SELECT * FROM iceberg.test_metadata_schema.iceberg_table1", "Table metadata is missing"); assertQuerySucceeds("DROP TABLE iceberg.test_metadata_schema.iceberg_table1"); assertQuery("SHOW TABLES FROM iceberg.test_metadata_schema", "VALUES 'iceberg_table2'"); } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestHiveTableOperationsConfig.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestHiveTableOperationsConfig.java new file mode 100644 index 0000000000000..db1b2d8d885f2 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestHiveTableOperationsConfig.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.hive; + +import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.units.Duration.succinctDuration; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestHiveTableOperationsConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(IcebergHiveTableOperationsConfig.class) + .setTableRefreshBackoffMinSleepTime(succinctDuration(100, MILLISECONDS)) + .setTableRefreshBackoffMaxSleepTime(succinctDuration(5, SECONDS)) + .setTableRefreshMaxRetryTime(succinctDuration(1, MINUTES)) + .setTableRefreshBackoffScaleFactor(4.0) + .setTableRefreshRetries(20)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("iceberg.hive.table-refresh.backoff-min-sleep-time", "10s") + .put("iceberg.hive.table-refresh.backoff-max-sleep-time", "20s") + .put("iceberg.hive.table-refresh.max-retry-time", "30s") + .put("iceberg.hive.table-refresh.retries", "42") + .put("iceberg.hive.table-refresh.backoff-scale-factor", "2.0") + .build(); + + IcebergHiveTableOperationsConfig expected = new IcebergHiveTableOperationsConfig() + .setTableRefreshBackoffMinSleepTime(succinctDuration(10, SECONDS)) + .setTableRefreshBackoffMaxSleepTime(succinctDuration(20, SECONDS)) + .setTableRefreshMaxRetryTime(succinctDuration(30, SECONDS)) + .setTableRefreshBackoffScaleFactor(2.0) + .setTableRefreshRetries(42); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java index 039ef121ac4e2..7bd7fb34bacee 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.IcebergDistributedTestBase; +import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; import com.facebook.presto.iceberg.IcebergUtil; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.spi.ConnectorId; @@ -66,6 +67,7 @@ protected Table loadTable(String tableName) return IcebergUtil.getHiveIcebergTable(getFileHiveMetastore(), getHdfsEnvironment(), + new IcebergHiveTableOperationsConfig(), getQueryRunner().getDefaultSession().toConnectorSession(connectorId), SchemaTableName.valueOf("tpch." + tableName)); } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java index 82061265b2f8f..d83095286dfa7 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java @@ -24,6 +24,7 @@ import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.IcebergConfig; import com.facebook.presto.iceberg.IcebergDistributedSmokeTestBase; +import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; import com.facebook.presto.iceberg.IcebergUtil; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; @@ -78,6 +79,7 @@ protected Table getIcebergTable(ConnectorSession session, String schema, String { return IcebergUtil.getHiveIcebergTable(getFileHiveMetastore(), getHdfsEnvironment(), + new IcebergHiveTableOperationsConfig(), session, SchemaTableName.valueOf(schema + "." + tableName)); }