diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index b8f2176118f7..88fd87489575 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -1177,7 +1177,14 @@ acceptedBreaks: old: "class org.apache.iceberg.Metrics" new: "class org.apache.iceberg.Metrics" justification: "Java serialization across versions is not guaranteed" + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.Table org.apache.iceberg.catalog.SessionCatalog::registerTable(org.apache.iceberg.catalog.SessionCatalog.SessionContext,\ + \ org.apache.iceberg.catalog.TableIdentifier, java.lang.String, boolean)" + justification: "New API with default implementation provided" org.apache.iceberg:iceberg-core: + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.rest.requests.RegisterTableRequest::overwrite()" + justification: "All subclasses implement the new method" - code: "java.method.removed" old: "method java.lang.String[] org.apache.iceberg.hadoop.Util::blockLocations(org.apache.iceberg.CombinedScanTask,\ \ org.apache.hadoop.conf.Configuration)" diff --git a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java index 897acd2e3ba6..437b15c5e105 100644 --- a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java +++ b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java @@ -344,6 +344,25 @@ default void invalidateTable(TableIdentifier identifier) {} * @throws AlreadyExistsException if the table already exists in the catalog. */ default Table registerTable(TableIdentifier identifier, String metadataFileLocation) { + return registerTable( + identifier, metadataFileLocation, false /* register only if it does not exist */); + } + + /** + * Register a table with the catalog, optionally overwrite existing table metadata. + * + *

Note: Overwriting an existing table may result in a change of table UUID, + * to match the one in the metadata file. + * + * @param identifier a table identifier + * @param metadataFileLocation the location of a metadata file + * @param overwrite if true, overwrite the existing table with provided metadata + * @return a Table instance + * @throws AlreadyExistsException if the table already exists in the catalog and overwrite is + * false. + */ + default Table registerTable( + TableIdentifier identifier, String metadataFileLocation, boolean overwrite) { throw new UnsupportedOperationException("Registering tables is not supported"); } diff --git a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java index fe29f8918531..9fd3c5ea0ebf 100644 --- a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java +++ b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java @@ -168,7 +168,30 @@ public Object wrappedIdentity() { * @return a Table instance * @throws AlreadyExistsException if the table already exists in the catalog. */ - Table registerTable(SessionContext context, TableIdentifier ident, String metadataFileLocation); + default Table registerTable( + SessionContext context, TableIdentifier ident, String metadataFileLocation) { + return registerTable(context, ident, metadataFileLocation, false); + } + + /** + * Register a table with the catalog, optionally overwrite existing table metadata. + * + *

Note: Overwriting an existing table may result in a change of table UUID, + * to match the one in the metadata file. + * + * @param context session context + * @param ident a table identifier + * @param metadataFileLocation the location of a metadata file + * @param overwrite if true, overwrite the existing table with provided metadata + * @return a Table instance + * @throws AlreadyExistsException if the table already exists in the catalog and overwrite is + * false. + */ + Table registerTable( + SessionContext context, + TableIdentifier ident, + String metadataFileLocation, + boolean overwrite); /** * Check whether table exists. diff --git a/core/src/main/java/org/apache/iceberg/BaseMetastoreCatalog.java b/core/src/main/java/org/apache/iceberg/BaseMetastoreCatalog.java index 940d7fa05ec6..68cada263cd8 100644 --- a/core/src/main/java/org/apache/iceberg/BaseMetastoreCatalog.java +++ b/core/src/main/java/org/apache/iceberg/BaseMetastoreCatalog.java @@ -26,7 +26,6 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.io.InputFile; import org.apache.iceberg.metrics.MetricsReporter; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -71,24 +70,43 @@ public Table loadTable(TableIdentifier identifier) { } @Override - public Table registerTable(TableIdentifier identifier, String metadataFileLocation) { + public Table registerTable( + TableIdentifier identifier, String metadataFileLocation, boolean overwrite) { Preconditions.checkArgument( identifier != null && isValidIdentifier(identifier), "Invalid identifier: %s", identifier); Preconditions.checkArgument( metadataFileLocation != null && !metadataFileLocation.isEmpty(), "Cannot register an empty metadata file location as a table"); - // Throw an exception if this table already exists in the catalog. - if (tableExists(identifier)) { + TableOperations ops = newTableOps(identifier); + + if (overwrite) { + return overwriteTable(identifier, metadataFileLocation, ops); + } else if (ops.current() == null) { + // create a new table + TableMetadata newMetadata = + TableMetadataParser.read(ops.io().newInputFile(metadataFileLocation)); + ops.commit(null, newMetadata); + return new BaseTable(ops, fullTableName(name(), identifier), metricsReporter()); + } else { throw new AlreadyExistsException("Table already exists: %s", identifier); } + } - TableOperations ops = newTableOps(identifier); - InputFile metadataFile = ops.io().newInputFile(metadataFileLocation); - TableMetadata metadata = TableMetadataParser.read(metadataFile); - ops.commit(null, metadata); + private Table overwriteTable( + TableIdentifier identifier, String metadataFileLocation, TableOperations ops) { + TableMetadata currentMetadata = ops.current(); - return new BaseTable(ops, fullTableName(name(), identifier), metricsReporter()); + if (currentMetadata != null + && currentMetadata.metadataFileLocation().equals(metadataFileLocation)) { + LOG.info( + "The requested metadata matches the existing metadata. No changes will be committed."); + return new BaseTable(ops, fullTableName(name(), identifier), metricsReporter()); + } + + // create or replace table metadata atomically + setAsCurrent(identifier, metadataFileLocation); + return loadTable(identifier); } @Override @@ -299,6 +317,13 @@ protected MetricsReporter metricsReporter() { return metricsReporter; } + // Atomically set the given metadata location as the current table metadata + protected void setAsCurrent(TableIdentifier identifier, String metadataLocation) { + throw new UnsupportedOperationException( + String.format( + "Overwrite table metadata on registration is not supported in %s catalog", name())); + } + @Override public void close() throws IOException { if (metricsReporter != null) { diff --git a/core/src/main/java/org/apache/iceberg/CachingCatalog.java b/core/src/main/java/org/apache/iceberg/CachingCatalog.java index 913f1a9482e1..08a969150604 100644 --- a/core/src/main/java/org/apache/iceberg/CachingCatalog.java +++ b/core/src/main/java/org/apache/iceberg/CachingCatalog.java @@ -191,8 +191,9 @@ public void invalidateTable(TableIdentifier ident) { } @Override - public Table registerTable(TableIdentifier identifier, String metadataFileLocation) { - Table table = catalog.registerTable(identifier, metadataFileLocation); + public Table registerTable( + TableIdentifier identifier, String metadataFileLocation, boolean overwrite) { + Table table = catalog.registerTable(identifier, metadataFileLocation, overwrite); invalidateTable(identifier); return table; } diff --git a/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java b/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java index d6ee4d345cfa..bbcafa7144f8 100644 --- a/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java @@ -85,8 +85,9 @@ public TableBuilder buildTable(TableIdentifier ident, Schema schema) { } @Override - public Table registerTable(TableIdentifier ident, String metadataFileLocation) { - return BaseSessionCatalog.this.registerTable(context, ident, metadataFileLocation); + public Table registerTable( + TableIdentifier ident, String metadataFileLocation, boolean overwrite) { + return BaseSessionCatalog.this.registerTable(context, ident, metadataFileLocation, overwrite); } @Override diff --git a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java index 067203b8d9eb..3bcc99faaeeb 100644 --- a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java +++ b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java @@ -108,6 +108,13 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) { defaultNamespaceLocation(tableIdentifier.namespace()), tableIdentifier.name()); } + @Override + protected void setAsCurrent(TableIdentifier identifier, String metadataLocation) { + synchronized (this) { + tables.put(identifier, metadataLocation); + } + } + private String defaultNamespaceLocation(Namespace namespace) { if (namespace.isEmpty()) { return warehouseLocation; diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java index 4d0aa08da2ba..570a14a40dc4 100644 --- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java +++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.sql.DataTruncation; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -28,6 +29,7 @@ import java.sql.SQLNonTransientConnectionException; import java.sql.SQLTimeoutException; import java.sql.SQLTransientConnectionException; +import java.sql.SQLWarning; import java.util.AbstractMap; import java.util.Arrays; import java.util.List; @@ -49,6 +51,7 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; @@ -283,6 +286,36 @@ protected String defaultWarehouseLocation(TableIdentifier table) { return SLASH.join(defaultNamespaceLocation(table.namespace()), table.name()); } + @Override + protected void setAsCurrent(TableIdentifier tableIdentifier, String metadataLocation) { + try { + int updatedRecords = + JdbcUtil.setMetadataLocationTable( + schemaVersion, connections, catalogName, tableIdentifier, metadataLocation); + if (updatedRecords == 1) { + LOG.debug( + "Successfully committed {} to existing table: {}", metadataLocation, tableIdentifier); + } else { + throw new CommitFailedException( + "Failed to update table %s to %s from catalog %s", + tableIdentifier, metadataLocation, catalogName); + } + } catch (SQLTimeoutException e) { + throw new UncheckedSQLException(e, "Database Connection timeout"); + } catch (SQLTransientConnectionException | SQLNonTransientConnectionException e) { + throw new UncheckedSQLException(e, "Database Connection failed"); + } catch (DataTruncation e) { + throw new UncheckedSQLException(e, "Database data truncation error"); + } catch (SQLWarning e) { + throw new UncheckedSQLException(e, "Database warning"); + } catch (SQLException e) { + throw new UncheckedSQLException(e, "Unknown failure"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException(e, "Interrupted during setAsCurrent"); + } + } + @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { TableOperations ops = newTableOps(identifier); diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java b/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java index 544e9f39c7cb..ee03796e65cd 100644 --- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java +++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java @@ -122,6 +122,57 @@ enum SchemaVersion { + " = ? AND " + JdbcTableOperations.METADATA_LOCATION_PROP + " = ?"; + private static final String V1_SET_METADATA_TABLE_SQL = + "UPDATE " + + CATALOG_TABLE_VIEW_NAME + + " SET " + + JdbcTableOperations.METADATA_LOCATION_PROP + + " = ?" + + " WHERE " + + CATALOG_NAME + + " = ? AND " + + TABLE_NAMESPACE + + " = ? AND " + + TABLE_NAME + + " = ? AND (" + + RECORD_TYPE + + " = '" + + TABLE_RECORD_TYPE + + "'" + + " OR " + + RECORD_TYPE + + " IS NULL)"; + private static final String V1_SET_METADATA_VIEW_SQL = + "UPDATE " + + CATALOG_TABLE_VIEW_NAME + + " SET " + + JdbcTableOperations.METADATA_LOCATION_PROP + + " = ?" + + " WHERE " + + CATALOG_NAME + + " = ? AND " + + TABLE_NAMESPACE + + " = ? AND " + + TABLE_NAME + + " = ? AND " + + RECORD_TYPE + + " = " + + "'" + + VIEW_RECORD_TYPE + + "'"; + private static final String V0_SET_METADATA_SQL = + "UPDATE " + + CATALOG_TABLE_VIEW_NAME + + " SET " + + JdbcTableOperations.METADATA_LOCATION_PROP + + " = ?" + + " WHERE " + + CATALOG_NAME + + " = ? AND " + + TABLE_NAMESPACE + + " = ? AND " + + TABLE_NAME + + " = ?"; static final String V0_CREATE_CATALOG_SQL = "CREATE TABLE " + CATALOG_TABLE_VIEW_NAME @@ -527,6 +578,32 @@ static Properties filterAndRemovePrefix(Map properties, String p return result; } + private static int setMetadataLocation( + boolean isTable, + SchemaVersion schemaVersion, + JdbcClientPool connections, + String catalogName, + TableIdentifier identifier, + String newMetadataLocation) + throws SQLException, InterruptedException { + return connections.run( + conn -> { + try (PreparedStatement sql = + conn.prepareStatement( + (schemaVersion == SchemaVersion.V1) + ? (isTable ? V1_SET_METADATA_TABLE_SQL : V1_SET_METADATA_VIEW_SQL) + : V0_SET_METADATA_SQL)) { + // UPDATE + sql.setString(1, newMetadataLocation); + // WHERE + sql.setString(2, catalogName); + sql.setString(3, namespaceToString(identifier.namespace())); + sql.setString(4, identifier.name()); + return sql.executeUpdate(); + } + }); + } + private static int update( boolean isTable, SchemaVersion schemaVersion, @@ -557,6 +634,17 @@ private static int update( }); } + static int setMetadataLocationTable( + SchemaVersion schemaVersion, + JdbcClientPool connections, + String catalogName, + TableIdentifier tableIdentifier, + String metadataLocation) + throws SQLException, InterruptedException { + return setMetadataLocation( + true, schemaVersion, connections, catalogName, tableIdentifier, metadataLocation); + } + static int updateTable( SchemaVersion schemaVersion, JdbcClientPool connections, diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index f4f076f970e7..2439483d64cc 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -294,7 +294,9 @@ public static LoadTableResponse registerTable( request.validate(); TableIdentifier identifier = TableIdentifier.of(namespace, request.name()); - Table table = catalog.registerTable(identifier, request.metadataLocation()); + Table table = + catalog.registerTable( + identifier, request.metadataLocation(), Boolean.TRUE.equals(request.overwrite())); if (table instanceof BaseTable) { return LoadTableResponse.builder() .withTableMetadata(((BaseTable) table).operations().current()) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java index 0176128ed576..c50f1468d398 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java @@ -222,6 +222,12 @@ public Table registerTable(TableIdentifier ident, String metadataFileLocation) { return delegate.registerTable(ident, metadataFileLocation); } + @Override + public Table registerTable( + TableIdentifier ident, String metadataFileLocation, boolean overwrite) { + return delegate.registerTable(ident, metadataFileLocation, overwrite); + } + @Override public void createNamespace(Namespace ns, Map props) { nsDelegate.createNamespace(ns, props); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 4692b92b51be..8952c743574b 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -499,7 +499,10 @@ public void invalidateTable(SessionContext context, TableIdentifier ident) {} @Override public Table registerTable( - SessionContext context, TableIdentifier ident, String metadataFileLocation) { + SessionContext context, + TableIdentifier ident, + String metadataFileLocation, + boolean overwrite) { Endpoint.check(endpoints, Endpoint.V1_REGISTER_TABLE); checkIdentifierIsValid(ident); @@ -512,6 +515,7 @@ public Table registerTable( ImmutableRegisterTableRequest.builder() .name(ident.name()) .metadataLocation(metadataFileLocation) + .overwrite(overwrite) .build(); AuthSession contextualSession = authManager.contextualSession(context, catalogAuth); diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequest.java b/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequest.java index 33b37dae242f..d3bd2764b301 100644 --- a/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequest.java +++ b/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequest.java @@ -28,6 +28,8 @@ public interface RegisterTableRequest extends RESTRequest { String metadataLocation(); + boolean overwrite(); + @Override default void validate() { // nothing to validate as it's not possible to create an invalid instance diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequestParser.java b/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequestParser.java index 961b6c185b87..726e7f9c777b 100644 --- a/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequestParser.java +++ b/core/src/main/java/org/apache/iceberg/rest/requests/RegisterTableRequestParser.java @@ -28,6 +28,7 @@ public class RegisterTableRequestParser { private static final String NAME = "name"; private static final String METADATA_LOCATION = "metadata-location"; + private static final String OVERWRITE = "overwrite"; private RegisterTableRequestParser() {} @@ -46,6 +47,7 @@ public static void toJson(RegisterTableRequest request, JsonGenerator gen) throw gen.writeStringField(NAME, request.name()); gen.writeStringField(METADATA_LOCATION, request.metadataLocation()); + gen.writeBooleanField(OVERWRITE, request.overwrite()); gen.writeEndObject(); } @@ -60,10 +62,12 @@ public static RegisterTableRequest fromJson(JsonNode json) { String name = JsonUtil.getString(NAME, json); String metadataLocation = JsonUtil.getString(METADATA_LOCATION, json); + boolean overwrite = Boolean.TRUE.equals(JsonUtil.getBoolOrNull(OVERWRITE, json)); return ImmutableRegisterTableRequest.builder() .name(name) .metadataLocation(metadataLocation) + .overwrite(overwrite) .build(); } } diff --git a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java index c2fd24856fb2..56a8a853b1c9 100644 --- a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg.catalog; +import static org.apache.iceberg.expressions.Expressions.bucket; import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -50,6 +51,8 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableUtil; @@ -192,6 +195,10 @@ protected boolean supportsEmptyNamespace() { return false; } + protected boolean supportsOverwriteRegistration() { + return false; + } + protected String baseTableLocation(TableIdentifier identifier) { return BASE_TABLE_LOCATION + "/" + identifier.namespace() + "/" + identifier.name(); } @@ -3139,6 +3146,10 @@ public void testRegisterTable() { TestHelpers.assertSameSchemaMap(registeredTable.schemas(), originalTable.schemas()); assertFiles(registeredTable, FILE_B, FILE_C); + assertThat(ops.refresh().metadataFileLocation()) + .as("metadataFileLocation must match") + .isEqualTo(metadataLocation); + registeredTable.newFastAppend().appendFile(FILE_A).commit(); assertFiles(registeredTable, FILE_B, FILE_C, FILE_A); @@ -3164,9 +3175,106 @@ public void testRegisterExistingTable() { assertThatThrownBy(() -> catalog.registerTable(identifier, metadataLocation)) .isInstanceOf(AlreadyExistsException.class) .hasMessageStartingWith("Table already exists: a.t1"); + assertThatThrownBy(() -> catalog.registerTable(identifier, metadataLocation, false)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageStartingWith("Table already exists: a.t1"); assertThat(catalog.dropTable(identifier)).isTrue(); } + @Test + public void testRegisterAndOverwriteForeignTable() { + assumeThat(supportsOverwriteRegistration()).isTrue(); + C catalog = catalog(); + + TableIdentifier identT1 = TableIdentifier.of("a", "t1"); + TableIdentifier identT2 = TableIdentifier.of("a", "t2"); + + if (requiresNamespaceCreate()) { + catalog.createNamespace(identT1.namespace()); + } + + Table t1 = catalog.createTable(identT1, SCHEMA); + UUID t1UUID = t1.uuid(); + Table t2 = + catalog.createTable( + identT2, SCHEMA, PartitionSpec.builderFor(SCHEMA).bucket("id", 16).build()); + assertThat(t1.spec().isPartitioned()).isFalse(); + assertThat(t2.spec().isPartitioned()).isTrue(); + + TableOperations opsT2 = operation(t2); + + // register table t1 with metadata from table t2 + Table registered = catalog.registerTable(identT1, opsT2.current().metadataFileLocation(), true); + + assertThat(registered.uuid()) + .as("table UUID should differ when registering with foreign metadata") + .isNotEqualTo(t1UUID); + assertThat(registered.spec().isPartitioned()) + .as("table is expected to be partitioned after registration with t2’s metadata") + .isEqualTo(t2.spec().isPartitioned()) + .isTrue(); + assertThat(operation(registered).refresh()) + .usingRecursiveComparison() + .as("TableMetadata fields must match") + .isEqualTo(opsT2.current()); + + // Both tables now point to the same metadata location, so we only need to purge once. + assertThat(catalog.dropTable(identT1, true)).isTrue(); + assertThat(catalog.dropTable(identT2, false)).isTrue(); + } + + private TableOperations operation(Table table) { + return ((BaseTable) table).operations(); + } + + @Test + public void testRegisterAndOverwriteExistingTable() { + assumeThat(supportsOverwriteRegistration()).isTrue(); + C catalog = catalog(); + + TableIdentifier ident = TableIdentifier.of("a", "e1"); + + if (requiresNamespaceCreate()) { + catalog.createNamespace(ident.namespace()); + } + + Table table = catalog.createTable(ident, SCHEMA); + UUID tableUUID = table.uuid(); + TableMetadata metadata = operation(table).current(); + String originalMetadataLocation = metadata.metadataFileLocation(); + // Serialize the table’s current (unpartitioned) metadata to JSON for later comparison. + String metadataAsJson = TableMetadataParser.toJson(metadata); + + // update table spec + table.updateSpec().addField(bucket("id", 16)).commit(); + assertThat(table.spec().isPartitioned()).isTrue(); + + // overwrite the table’s metadata with its original unpartitioned version. + Table overwritten = catalog.registerTable(ident, originalMetadataLocation, true); + + assertThat(overwritten.uuid()) + .as("UUID should remain the same when overwriting an existing table's own metadata") + .isEqualTo(tableUUID); + assertThat(overwritten.spec().isPartitioned()) + .as("table is expected to be unpartitioned") + .isFalse(); + + TableMetadata actual = operation(overwritten).refresh(); + TableMetadata expected = TableMetadataParser.fromJson(metadataAsJson); + assertThat(actual.metadataFileLocation()) + .as("metadataFileLocation must match") + .isEqualTo(originalMetadataLocation); + assertThat(actual) + .usingRecursiveComparison() + // reason to ignore: + // TableMetadataParser to/fromJson skips recording of metadataFileLocation in TableMetadata + .ignoringFields("metadataFileLocation") + .as("tableMetadata fields must match") + .isEqualTo(expected); + + assertThat(catalog.dropTable(ident)).isTrue(); + } + @Test public void testCatalogWithCustomMetricsReporter() throws IOException { C catalogWithCustomReporter = diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java index 61f5db9029b4..976fa8c4f7bf 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java @@ -20,6 +20,7 @@ import static org.apache.iceberg.NullOrder.NULLS_FIRST; import static org.apache.iceberg.SortDirection.ASC; +import static org.apache.iceberg.expressions.Expressions.bucket; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -676,6 +677,28 @@ public void testRegisterExistingTable() throws IOException { assertThatThrownBy(() -> catalog.registerTable(identifier, metadataLocation)) .isInstanceOf(AlreadyExistsException.class) .hasMessage("Table already exists: a.t1"); + assertThatThrownBy(() -> catalog.registerTable(identifier, metadataLocation, false)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessage("Table already exists: a.t1"); + assertThat(catalog.dropTable(identifier)).isTrue(); + } + + @Test + public void testRegisterAndOverwriteExistingTable() throws IOException { + TableIdentifier identifier = TableIdentifier.of("a", "t1"); + HadoopCatalog catalog = hadoopCatalog(); + catalog.createTable(identifier, SCHEMA); + Table registeringTable = catalog.loadTable(identifier); + TableOperations ops = ((HasTableOperations) registeringTable).operations(); + String unpartitionedMetadataLocation = ops.current().metadataFileLocation(); + // update table spec + registeringTable.updateSpec().addField(bucket("id", 16)).commit(); + assertThat(registeringTable.spec().isPartitioned()).isTrue(); + // register with overwrite + assertThatThrownBy(() -> catalog.registerTable(identifier, unpartitionedMetadataLocation, true)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Overwrite table metadata on registration is not supported in hadoop catalog"); + assertThat(catalog.dropTable(identifier)).isTrue(); } } diff --git a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java index 705ff3dc8699..45f15757636e 100644 --- a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java +++ b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java @@ -71,4 +71,9 @@ protected boolean requiresNamespaceCreate() { protected boolean supportsEmptyNamespace() { return true; } + + @Override + protected boolean supportsOverwriteRegistration() { + return true; + } } diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java index 0b7315787a26..2411d9b2320d 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java @@ -112,6 +112,11 @@ protected boolean supportsEmptyNamespace() { return true; } + @Override + protected boolean supportsOverwriteRegistration() { + return true; + } + @Override protected boolean supportsNamesWithDot() { // namespaces with a dot are not supported diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalogWithV1Schema.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalogWithV1Schema.java index 88514ac90523..18404c86ea05 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalogWithV1Schema.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalogWithV1Schema.java @@ -71,6 +71,11 @@ protected boolean supportsNamesWithDot() { return false; } + @Override + protected boolean supportsOverwriteRegistration() { + return true; + } + @Override protected boolean supportsNestedNamespaces() { return true; diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcUtil.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcUtil.java index 4ac3a9301b4a..f0ca1a68bd14 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcUtil.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcUtil.java @@ -33,6 +33,62 @@ public class TestJdbcUtil { + private static final String CATALOG_NAME = "TEST"; + private static final String NAMESPACE1 = "namespace1"; + private static final Namespace TEST_NAMESPACE = Namespace.of(NAMESPACE1); + private static final String OLD_LOCATION = "testLocation"; + private static final String NEW_LOCATION = "newLocation"; + + private String createTempDatabase() throws Exception { + java.nio.file.Path dbFile = Files.createTempFile("icebergSchemaUpdate", "db"); + return "jdbc:sqlite:" + dbFile.toAbsolutePath(); + } + + private JdbcClientPool setupConnectionPool(String jdbcUrl) { + return new JdbcClientPool(jdbcUrl, Maps.newHashMap()); + } + + // Initializes the database with V0 schema and creates sample tables. + private void setupV0SchemaWithTables(JdbcClientPool connections) throws Exception { + connections.newClient().prepareStatement(JdbcUtil.V0_CREATE_CATALOG_SQL).executeUpdate(); + + // inserting tables + JdbcUtil.doCommitCreateTable( + JdbcUtil.SchemaVersion.V0, + connections, + CATALOG_NAME, + TEST_NAMESPACE, + TableIdentifier.of(TEST_NAMESPACE, "table1"), + OLD_LOCATION); + JdbcUtil.doCommitCreateTable( + JdbcUtil.SchemaVersion.V0, + connections, + CATALOG_NAME, + TEST_NAMESPACE, + TableIdentifier.of(TEST_NAMESPACE, "table2"), + OLD_LOCATION); + } + + private void upgradeSchemaV0ToV1(JdbcClientPool connections) throws Exception { + connections.newClient().prepareStatement(JdbcUtil.V1_UPDATE_CATALOG_SQL).execute(); + } + + /** Verifies that the expected tables exist in the database with the correct metadata. */ + private void verifyTablesExist(JdbcClientPool connections, String... expectedTableNames) + throws Exception { + try (PreparedStatement statement = + connections.newClient().prepareStatement(JdbcUtil.V0_LIST_TABLE_SQL)) { + statement.setString(1, CATALOG_NAME); + statement.setString(2, NAMESPACE1); + ResultSet tables = statement.executeQuery(); + + for (String expectedTableName : expectedTableNames) { + assertThat(tables.next()).isTrue(); + assertThat(tables.getString(JdbcUtil.TABLE_NAME)).isEqualTo(expectedTableName); + } + } + } + @Test public void testFilterAndRemovePrefix() { Map input = Maps.newHashMap(); @@ -54,60 +110,34 @@ public void testFilterAndRemovePrefix() { @Test public void testV0toV1SqlStatements() throws Exception { - java.nio.file.Path dbFile = Files.createTempFile("icebergSchemaUpdate", "db"); - String jdbcUrl = "jdbc:sqlite:" + dbFile.toAbsolutePath(); - + String jdbcUrl = createTempDatabase(); SQLiteDataSource dataSource = new SQLiteDataSource(); dataSource.setUrl(jdbcUrl); - try (JdbcClientPool connections = new JdbcClientPool(jdbcUrl, Maps.newHashMap())) { + try (JdbcClientPool connections = setupConnectionPool(jdbcUrl)) { // create "old style" SQL schema - connections.newClient().prepareStatement(JdbcUtil.V0_CREATE_CATALOG_SQL).executeUpdate(); - - // inserting tables - JdbcUtil.doCommitCreateTable( - JdbcUtil.SchemaVersion.V0, - connections, - "TEST", - Namespace.of("namespace1"), - TableIdentifier.of(Namespace.of("namespace1"), "table1"), - "testLocation"); - JdbcUtil.doCommitCreateTable( - JdbcUtil.SchemaVersion.V0, - connections, - "TEST", - Namespace.of("namespace1"), - TableIdentifier.of(Namespace.of("namespace1"), "table2"), - "testLocation"); + setupV0SchemaWithTables(connections); - try (PreparedStatement statement = - connections.newClient().prepareStatement(JdbcUtil.V0_LIST_TABLE_SQL)) { - statement.setString(1, "TEST"); - statement.setString(2, "namespace1"); - ResultSet tables = statement.executeQuery(); - tables.next(); - assertThat(tables.getString(JdbcUtil.TABLE_NAME)).isEqualTo("table1"); - tables.next(); - assertThat(tables.getString(JdbcUtil.TABLE_NAME)).isEqualTo("table2"); - } + // Verify tables exist + verifyTablesExist(connections, "table1", "table2"); // updating the schema from V0 to V1 - connections.newClient().prepareStatement(JdbcUtil.V1_UPDATE_CATALOG_SQL).execute(); + upgradeSchemaV0ToV1(connections); // trying to add a table on the updated schema JdbcUtil.doCommitCreateTable( JdbcUtil.SchemaVersion.V1, connections, - "TEST", - Namespace.of("namespace1"), - TableIdentifier.of(Namespace.of("namespace1"), "table3"), - "testLocation"); + CATALOG_NAME, + TEST_NAMESPACE, + TableIdentifier.of(TEST_NAMESPACE, "table3"), + OLD_LOCATION); // testing the tables after migration and new table added try (PreparedStatement statement = connections.newClient().prepareStatement(JdbcUtil.V0_LIST_TABLE_SQL)) { - statement.setString(1, "TEST"); - statement.setString(2, "namespace1"); + statement.setString(1, CATALOG_NAME); + statement.setString(2, NAMESPACE1); ResultSet tables = statement.executeQuery(); tables.next(); assertThat(tables.getString(JdbcUtil.TABLE_NAME)).isEqualTo("table1"); @@ -125,10 +155,10 @@ public void testV0toV1SqlStatements() throws Exception { JdbcUtil.updateTable( JdbcUtil.SchemaVersion.V1, connections, - "TEST", - TableIdentifier.of(Namespace.of("namespace1"), "table3"), - "newLocation", - "testLocation"); + CATALOG_NAME, + TableIdentifier.of(TEST_NAMESPACE, "table3"), + NEW_LOCATION, + OLD_LOCATION); assertThat(updated).isEqualTo(1); // update a table (commit) migrated from V0 schema @@ -136,11 +166,80 @@ public void testV0toV1SqlStatements() throws Exception { JdbcUtil.updateTable( JdbcUtil.SchemaVersion.V1, connections, - "TEST", - TableIdentifier.of(Namespace.of("namespace1"), "table1"), - "newLocation", - "testLocation"); + CATALOG_NAME, + TableIdentifier.of(TEST_NAMESPACE, "table1"), + NEW_LOCATION, + OLD_LOCATION); + assertThat(updated).isEqualTo(1); + } + } + + @Test + public void testSetMetadataLocationTable() throws Exception { + String jdbcUrl = createTempDatabase(); + + try (JdbcClientPool connections = setupConnectionPool(jdbcUrl)) { + // Test with V0 schema + setupV0SchemaWithTables(connections); + + TableIdentifier table1 = TableIdentifier.of(TEST_NAMESPACE, "table1"); + TableIdentifier table2 = TableIdentifier.of(TEST_NAMESPACE, "table2"); + + // Test setMetadataLocationTable with V0 schema + int updated = + JdbcUtil.setMetadataLocationTable( + JdbcUtil.SchemaVersion.V0, connections, CATALOG_NAME, table1, NEW_LOCATION); + assertThat(updated).isEqualTo(1); + + // Verify the metadata location was updated + Map tableData = + JdbcUtil.loadTable(JdbcUtil.SchemaVersion.V0, connections, CATALOG_NAME, table1); + assertThat(tableData.get("metadata_location")).isEqualTo(NEW_LOCATION); + + // Upgrade to V1 schema + upgradeSchemaV0ToV1(connections); + + // Add a new table in V1 schema + TableIdentifier table3 = TableIdentifier.of(TEST_NAMESPACE, "table3"); + JdbcUtil.doCommitCreateTable( + JdbcUtil.SchemaVersion.V1, + connections, + CATALOG_NAME, + TEST_NAMESPACE, + table3, + OLD_LOCATION); + + // Test setMetadataLocationTable with V1 schema + String anotherNewLocation = "anotherNewLocation"; + updated = + JdbcUtil.setMetadataLocationTable( + JdbcUtil.SchemaVersion.V1, connections, CATALOG_NAME, table3, anotherNewLocation); + assertThat(updated).isEqualTo(1); + + // Verify the metadata location was updated for V1 table + tableData = JdbcUtil.loadTable(JdbcUtil.SchemaVersion.V1, connections, CATALOG_NAME, table3); + assertThat(tableData.get("metadata_location")).isEqualTo(anotherNewLocation); + + // Test setMetadataLocationTable on a migrated table (table2) with V1 schema + updated = + JdbcUtil.setMetadataLocationTable( + JdbcUtil.SchemaVersion.V1, connections, CATALOG_NAME, table2, anotherNewLocation); assertThat(updated).isEqualTo(1); + + // Verify the metadata location was updated for migrated table + tableData = JdbcUtil.loadTable(JdbcUtil.SchemaVersion.V1, connections, CATALOG_NAME, table2); + assertThat(tableData.get("metadata_location")).isEqualTo(anotherNewLocation); + + // Test set metadata location for non-existent table + TableIdentifier nonExistentTable = TableIdentifier.of(TEST_NAMESPACE, "nonexistent"); + updated = + JdbcUtil.setMetadataLocationTable( + JdbcUtil.SchemaVersion.V1, + connections, + CATALOG_NAME, + nonExistentTable, + "someLocation"); + assertThat(updated).isEqualTo(0); } } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java index 42983a1a6932..3533684990d3 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java @@ -284,6 +284,11 @@ protected boolean requiresNamespaceCreate() { return true; } + @Override + protected boolean supportsOverwriteRegistration() { + return true; + } + /* RESTCatalog specific tests */ @Test diff --git a/core/src/test/java/org/apache/iceberg/rest/requests/TestRegisterTableRequestParser.java b/core/src/test/java/org/apache/iceberg/rest/requests/TestRegisterTableRequestParser.java index 50a47df974a3..c2acbd190d5e 100644 --- a/core/src/test/java/org/apache/iceberg/rest/requests/TestRegisterTableRequestParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/requests/TestRegisterTableRequestParser.java @@ -62,12 +62,14 @@ public void roundTripSerde() { .name("table_1") .metadataLocation( "file://tmp/NS/test_tbl/metadata/00000-d4f60d2f-2ad2-408b-8832-0ed7fbd851ee.metadata.json") + .overwrite(true) .build(); String expectedJson = "{\n" + " \"name\" : \"table_1\",\n" - + " \"metadata-location\" : \"file://tmp/NS/test_tbl/metadata/00000-d4f60d2f-2ad2-408b-8832-0ed7fbd851ee.metadata.json\"\n" + + " \"metadata-location\" : \"file://tmp/NS/test_tbl/metadata/00000-d4f60d2f-2ad2-408b-8832-0ed7fbd851ee.metadata.json\",\n" + + " \"overwrite\" : true\n" + "}"; String json = RegisterTableRequestParser.toJson(request, true); @@ -76,4 +78,23 @@ public void roundTripSerde() { assertThat(RegisterTableRequestParser.toJson(RegisterTableRequestParser.fromJson(json), true)) .isEqualTo(expectedJson); } + + @Test + public void serdeOnDefaultAndExplicitOverwriteField() { + String defaultJson = + "{\n" + + " \"name\" : \"table_1\",\n" + + " \"metadata-location\" : \"file://tmp/NS/test_tbl/metadata/00000-d4f60d2f-2ad2-408b-8832-0ed7fbd851ee.metadata.json\"\n" + + "}"; + RegisterTableRequest defaultRequest = RegisterTableRequestParser.fromJson(defaultJson); + assertThat(defaultRequest.overwrite()).isFalse(); + String explicitJson = + "{\n" + + " \"name\" : \"table_1\",\n" + + " \"metadata-location\" : \"file://tmp/NS/test_tbl/metadata/00000-d4f60d2f-2ad2-408b-8832-0ed7fbd851ee.metadata.json\",\n" + + " \"overwrite\" :false\n" + + "}"; + RegisterTableRequest explicitRequest = RegisterTableRequestParser.fromJson(explicitJson); + assertThat(explicitRequest.overwrite()).isFalse(); + } } diff --git a/dell/src/test/java/org/apache/iceberg/dell/ecs/TestEcsCatalog.java b/dell/src/test/java/org/apache/iceberg/dell/ecs/TestEcsCatalog.java index 4714d37d72b9..9deee0c1f54c 100644 --- a/dell/src/test/java/org/apache/iceberg/dell/ecs/TestEcsCatalog.java +++ b/dell/src/test/java/org/apache/iceberg/dell/ecs/TestEcsCatalog.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.Map; +import java.util.UUID; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Schema; @@ -199,6 +200,13 @@ public void testRegisterExistingTable() { assertThatThrownBy(() -> ecsCatalog.registerTable(identifier, metadataLocation)) .isInstanceOf(AlreadyExistsException.class) .hasMessage("Table already exists: a.t1"); + assertThatThrownBy(() -> ecsCatalog.registerTable(identifier, metadataLocation, false)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessage("Table already exists: a.t1"); + assertThatThrownBy( + () -> ecsCatalog.registerTable(identifier, metadataLocation + UUID.randomUUID(), true)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Overwrite table metadata on registration is not supported in test catalog"); assertThat(ecsCatalog.dropTable(identifier, true)).isTrue(); } } diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index b0d9e224634d..d913122a6f05 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configurable; @@ -211,6 +212,28 @@ public String name() { return name; } + @Override + protected void setAsCurrent(TableIdentifier identifier, String metadataLocation) { + String database = identifier.namespace().level(0); + try { + Table tbl = clients.run(client -> client.getTable(database, identifier.name())); + Map parameters = + Optional.ofNullable(tbl.getParameters()).orElseGet(Maps::newHashMap); + parameters.put(BaseMetastoreTableOperations.METADATA_LOCATION_PROP, metadataLocation); + clients.run( + client -> { + MetastoreUtil.alterTable(client, database, identifier.name(), tbl, ImmutableMap.of()); + return null; + }); + } catch (TException e) { + throw new RuntimeException( + String.format("Failed to set %s as current in %s", metadataLocation, identifier), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted in call to setAsCurrent", e); + } + } + @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { if (!isValidIdentifier(identifier)) { diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveTable.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveTable.java index 031015ec5a0a..81601a683128 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveTable.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveTable.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -66,7 +67,6 @@ import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchTableException; @@ -624,12 +624,49 @@ public void testRegisterExistingTable() throws TException { assertThat(metadataVersionFiles).hasSize(1); // Try to register an existing table - assertThatThrownBy( - () -> catalog.registerTable(TABLE_IDENTIFIER, "file:" + metadataVersionFiles.get(0))) - .isInstanceOf(AlreadyExistsException.class) + String metadataLocation = "file:" + metadataVersionFiles.get(0); + assertThatThrownBy(() -> catalog.registerTable(TABLE_IDENTIFIER, metadataLocation)) + .hasMessage("Table already exists: hivedb.tbl"); + assertThatThrownBy(() -> catalog.registerTable(TABLE_IDENTIFIER, metadataLocation, false)) .hasMessage("Table already exists: hivedb.tbl"); } + @Test + public void testRegisterAndOverwriteExistingTable() throws TException { + org.apache.hadoop.hive.metastore.api.Table originalTable = + HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, TABLE_NAME); + Table icebergTable = catalog.loadTable(TABLE_IDENTIFIER); + String originalMetadataFilePath = + ((BaseTable) icebergTable).operations().refresh().metadataFileLocation(); + + assertThat(originalTable.getParameters()) + .isNotNull() + .containsEntry(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ROOT)) + .doesNotContainKey(PREVIOUS_METADATA_LOCATION_PROP) + .containsEntry(METADATA_LOCATION_PROP, originalMetadataFilePath); + assertThat(originalTable.getTableType()).isEqualToIgnoringCase("EXTERNAL_TABLE"); + + // drop the partition field + icebergTable.updateSpec().removeField("id").commit(); + org.apache.hadoop.hive.metastore.api.Table unpartitionedTable = + HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, TABLE_NAME); + String unpartitionedMetadataFilePath = + ((BaseTable) icebergTable).operations().refresh().metadataFileLocation(); + assertThat(unpartitionedTable.getParameters()) + .isNotNull() + .containsEntry(METADATA_LOCATION_PROP, unpartitionedMetadataFilePath) + .containsEntry(PREVIOUS_METADATA_LOCATION_PROP, originalMetadataFilePath); + + // register original + catalog.registerTable(TABLE_IDENTIFIER, originalMetadataFilePath, true /* overwrite */); + org.apache.hadoop.hive.metastore.api.Table overwrittenTable = + HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB_NAME, TABLE_NAME); + assertThat(overwrittenTable.getParameters()) + .containsEntry(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ROOT)) + .containsEntry(METADATA_LOCATION_PROP, originalMetadataFilePath); + assertThat(overwrittenTable.getSd()).isEqualTo(originalTable.getSd()); + } + @Test public void testEngineHiveEnabledDefault() throws TException { // Drop the previously created table to make place for the new one diff --git a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java index 87ec90663db2..91151b33aa7d 100644 --- a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java +++ b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java @@ -97,4 +97,9 @@ protected boolean supportsNamesWithDot() { return PropertyUtil.propertyAsBoolean( restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_NAMES_WITH_DOT, false); } + + @Override + protected boolean supportsOverwriteRegistration() { + return true; + } }