diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 6aaf0265e1a7..59ce870b5a1e 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -40,6 +40,13 @@ private TableProperties() { */ public static final String FORMAT_VERSION = "format-version"; + /** + * Reserved table property for UUID. + *

+ * This reserved property is used to store the UUID of the table. + */ + public static final String UUID = "uuid"; + /** * Reserved Iceberg table properties list. *

@@ -47,7 +54,8 @@ private TableProperties() { * The value of these properties are not persisted as a part of the table metadata. */ public static final Set RESERVED_PROPERTIES = ImmutableSet.of( - FORMAT_VERSION + FORMAT_VERSION, + UUID ); public static final String COMMIT_NUM_RETRIES = "commit.retry.num-retries"; diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 97f2d98bac05..a5e5efd72c98 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -884,5 +884,11 @@ public void testNoReservedPropertyForTableMetadataCreation() { "Table properties should not contain reserved properties, but got {format-version=1}", () -> TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), null, "/tmp", ImmutableMap.of(TableProperties.FORMAT_VERSION, "1"), 1)); + + AssertHelpers.assertThrows("should not allow reserved table property when creating table metadata", + IllegalArgumentException.class, + "Table properties should not contain reserved properties, but got {uuid=uuid}", + () -> TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), null, "/tmp", + ImmutableMap.of(TableProperties.UUID, "uuid"), 1)); } } diff --git a/flink/v1.12/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.12/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 7b552fa9ecb3..41908ce39177 100644 --- a/flink/v1.12/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v1.12/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.flink.table.api.DataTypes; @@ -178,10 +177,9 @@ public void testCreateTableIfNotExists() { sql("CREATE TABLE IF NOT EXISTS tl(id BIGINT)"); Assert.assertEquals(Maps.newHashMap(), table("tl").properties()); - final String uuid = UUID.randomUUID().toString(); - final Map expectedProperties = ImmutableMap.of("uuid", uuid); + final Map expectedProperties = ImmutableMap.of("key", "value"); table("tl").updateProperties() - .set("uuid", uuid) + .set("key", "value") .commit(); Assert.assertEquals(expectedProperties, table("tl").properties()); diff --git a/flink/v1.13/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.13/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 4ecfcb040cbb..b0efe154ce53 100644 --- a/flink/v1.13/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v1.13/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.flink.table.api.DataTypes; @@ -179,10 +178,9 @@ public void testCreateTableIfNotExists() { sql("CREATE TABLE IF NOT EXISTS tl(id BIGINT)"); Assert.assertEquals(Maps.newHashMap(), table("tl").properties()); - final String uuid = UUID.randomUUID().toString(); - final Map expectedProperties = ImmutableMap.of("uuid", uuid); + final Map expectedProperties = ImmutableMap.of("key", "value"); table("tl").updateProperties() - .set("uuid", uuid) + .set("key", "value") .commit(); Assert.assertEquals(expectedProperties, table("tl").properties()); diff --git a/flink/v1.14/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.14/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 4ecfcb040cbb..b0efe154ce53 100644 --- a/flink/v1.14/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v1.14/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.flink.table.api.DataTypes; @@ -179,10 +178,9 @@ public void testCreateTableIfNotExists() { sql("CREATE TABLE IF NOT EXISTS tl(id BIGINT)"); Assert.assertEquals(Maps.newHashMap(), table("tl").properties()); - final String uuid = UUID.randomUUID().toString(); - final Map expectedProperties = ImmutableMap.of("uuid", uuid); + final Map expectedProperties = ImmutableMap.of("key", "value"); table("tl").updateProperties() - .set("uuid", uuid) + .set("key", "value") .commit(); Assert.assertEquals(expectedProperties, table("tl").properties()); diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java index a0bb5c3d5963..f1f23a14549d 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java @@ -262,7 +262,7 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) { Map summary = Optional.ofNullable(metadata.currentSnapshot()) .map(Snapshot::summary) .orElseGet(ImmutableMap::of); - setHmsTableParameters(newMetadataLocation, tbl, metadata.properties(), removedProps, hiveEngineEnabled, summary); + setHmsTableParameters(newMetadataLocation, tbl, metadata, removedProps, hiveEngineEnabled, summary); if (!keepHiveStats) { tbl.getParameters().remove(StatsSetupConst.COLUMN_STATS_ACCURATE); @@ -352,18 +352,21 @@ private Table newHmsTable() { return newTable; } - private void setHmsTableParameters(String newMetadataLocation, Table tbl, Map icebergTableProps, + private void setHmsTableParameters(String newMetadataLocation, Table tbl, TableMetadata metadata, Set obsoleteProps, boolean hiveEngineEnabled, Map summary) { Map parameters = Optional.ofNullable(tbl.getParameters()) .orElseGet(Maps::newHashMap); // push all Iceberg table properties into HMS - icebergTableProps.forEach((key, value) -> { + metadata.properties().forEach((key, value) -> { // translate key names between Iceberg and HMS where needed String hmsKey = ICEBERG_TO_HMS_TRANSLATION.getOrDefault(key, key); parameters.put(hmsKey, value); }); + if (metadata.uuid() != null) { + parameters.put(TableProperties.UUID, metadata.uuid()); + } // remove any props from HMS that are no longer present in Iceberg table props obsoleteProps.forEach(parameters::remove); diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java index dea77be5c9b8..74716b771cb0 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java @@ -28,6 +28,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -442,4 +443,29 @@ private String defaultUri(Namespace namespace) throws TException { "hive.metastore.warehouse.dir", "") + "/" + namespace.level(0) + ".db"; } + @Test + public void testUUIDinTableProperties() throws Exception { + Schema schema = new Schema( + required(1, "id", Types.IntegerType.get(), "unique ID"), + required(2, "data", Types.StringType.get()) + ); + TableIdentifier tableIdentifier = TableIdentifier.of(DB_NAME, "tbl"); + String location = temp.newFolder("tbl").toString(); + + try { + catalog.buildTable(tableIdentifier, schema) + .withLocation(location) + .create(); + + String tableName = tableIdentifier.name(); + org.apache.hadoop.hive.metastore.api.Table hmsTable = + metastoreClient.getTable(tableIdentifier.namespace().level(0), tableName); + + // check parameters are in expected state + Map parameters = hmsTable.getParameters(); + Assert.assertNotNull(parameters.get(TableProperties.UUID)); + } finally { + catalog.dropTable(tableIdentifier); + } + } } diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index 1df2d28c0b06..938ba1997b28 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -624,7 +624,7 @@ public void testIcebergAndHmsTableProperties() throws Exception { Assert.assertEquals(expectedIcebergProperties, icebergTable.properties()); if (Catalogs.hiveCatalog(shell.getHiveConf(), tableProperties)) { - Assert.assertEquals(10, hmsParams.size()); + Assert.assertEquals(11, hmsParams.size()); Assert.assertEquals("initial_val", hmsParams.get("custom_property")); Assert.assertEquals("TRUE", hmsParams.get(InputFormatConfig.EXTERNAL_TABLE_PURGE)); Assert.assertEquals("TRUE", hmsParams.get("EXTERNAL")); @@ -662,7 +662,7 @@ public void testIcebergAndHmsTableProperties() throws Exception { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); if (Catalogs.hiveCatalog(shell.getHiveConf(), tableProperties)) { - Assert.assertEquals(13, hmsParams.size()); // 2 newly-added properties + previous_metadata_location prop + Assert.assertEquals(14, hmsParams.size()); // 2 newly-added properties + previous_metadata_location prop Assert.assertEquals("true", hmsParams.get("new_prop_1")); Assert.assertEquals("false", hmsParams.get("new_prop_2")); Assert.assertEquals("new_val", hmsParams.get("custom_property"));