diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java index 6cab1ab8cf69a..e88b75ba8bf14 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java @@ -42,6 +42,7 @@ import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slice; import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFiles; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; @@ -63,6 +64,7 @@ import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.iceberg.IcebergColumnHandle.primitiveIcebergColumnHandle; import static com.facebook.presto.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; +import static com.facebook.presto.iceberg.IcebergTableProperties.FORMAT_VERSION; import static com.facebook.presto.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static com.facebook.presto.iceberg.IcebergUtil.getColumns; import static com.facebook.presto.iceberg.IcebergUtil.getFileFormat; @@ -282,6 +284,10 @@ protected ImmutableMap createMetadataProperties(org.apache.icebe { ImmutableMap.Builder properties = ImmutableMap.builder(); properties.put(FILE_FORMAT_PROPERTY, getFileFormat(icebergTable)); + + int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + properties.put(FORMAT_VERSION, String.valueOf(formatVersion)); + if (!icebergTable.spec().fields().isEmpty()) { properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec())); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java index edf89407b5821..97eadcceeb192 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java @@ -59,6 +59,7 @@ import static com.facebook.presto.hive.HiveMetadata.TABLE_COMMENT; import static com.facebook.presto.iceberg.IcebergSchemaProperties.getSchemaLocation; import static com.facebook.presto.iceberg.IcebergTableProperties.getFileFormat; +import static com.facebook.presto.iceberg.IcebergTableProperties.getFormatVersion; import static com.facebook.presto.iceberg.IcebergTableProperties.getPartitioning; import static com.facebook.presto.iceberg.IcebergTableProperties.getTableLocation; import static com.facebook.presto.iceberg.IcebergUtil.getColumns; @@ -79,6 +80,7 @@ import static java.util.stream.Collectors.toList; import static org.apache.iceberg.TableMetadata.newTableMetadata; import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT; +import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.OBJECT_STORE_PATH; import static org.apache.iceberg.TableProperties.WRITE_DATA_LOCATION; import static org.apache.iceberg.TableProperties.WRITE_METADATA_LOCATION; @@ -263,13 +265,18 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con throw new TableAlreadyExistsException(schemaTableName); } - ImmutableMap.Builder propertiesBuilder = ImmutableMap.builderWithExpectedSize(2); + ImmutableMap.Builder propertiesBuilder = ImmutableMap.builderWithExpectedSize(3); FileFormat fileFormat = getFileFormat(tableMetadata.getProperties()); propertiesBuilder.put(DEFAULT_FILE_FORMAT, fileFormat.toString()); if (tableMetadata.getComment().isPresent()) { propertiesBuilder.put(TABLE_COMMENT, tableMetadata.getComment().get()); } + String formatVersion = getFormatVersion(tableMetadata.getProperties()); + if (formatVersion != null) { + propertiesBuilder.put(FORMAT_VERSION, formatVersion); + } + TableMetadata metadata = newTableMetadata(schema, partitionSpec, targetPath, propertiesBuilder.build()); transaction = createTableTransaction(tableName, operations, metadata); diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java index c1773b0c6650f..56de02191fdbc 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java @@ -84,23 +84,24 @@ public void testDescribeTable() MaterializedResult actualColumns = computeActual("DESCRIBE orders"); Assert.assertEquals(actualColumns, expectedColumns); } - + @Test public void testShowCreateTable() { assertThat(computeActual("SHOW CREATE TABLE orders").getOnlyValue()) .isEqualTo("CREATE TABLE iceberg.tpch.orders (\n" + - " orderkey bigint,\n" + - " custkey bigint,\n" + - " orderstatus varchar,\n" + - " totalprice double,\n" + - " orderdate date,\n" + - " orderpriority varchar,\n" + - " clerk varchar,\n" + - " shippriority integer,\n" + - " comment varchar\n" + + " \"orderkey\" bigint,\n" + + " \"custkey\" bigint,\n" + + " \"orderstatus\" varchar,\n" + + " \"totalprice\" double,\n" + + " \"orderdate\" date,\n" + + " \"orderpriority\" varchar,\n" + + " \"clerk\" varchar,\n" + + " \"shippriority\" integer,\n" + + " \"comment\" varchar\n" + ")\n" + "WITH (\n" + - " format = 'ORC'\n" + + " format = 'PARQUET',\n" + + " format_version = '1'\n" + ")"); } @@ -348,6 +349,7 @@ private void testCreatePartitionedTableAs(Session session, FileFormat fileFormat ")\n" + "WITH (\n" + " format = '" + fileFormat + "',\n" + + " format_version = '1',\n" + " partitioning = ARRAY['order_status','ship_priority','bucket(order_key, 9)']\n" + ")", getSession().getCatalog().get(), @@ -389,7 +391,8 @@ public void testTableComments() ")\n" + "COMMENT '%s'\n" + "WITH (\n" + - " format = 'ORC'\n" + + " format = 'ORC',\n" + + " format_version = '1'\n" + ")"; String createTableSql = format(createTableTemplate, "test table comment"); assertUpdate(createTableSql); @@ -474,6 +477,7 @@ private void testCreateTableLike() assertUpdate(session, "CREATE TABLE test_create_table_like_original (col1 INTEGER, aDate DATE) WITH(format = 'PARQUET', partitioning = ARRAY['aDate'])"); assertEquals(getTablePropertiesString("test_create_table_like_original"), "WITH (\n" + " format = 'PARQUET',\n" + + " format_version = '1',\n" + " partitioning = ARRAY['adate']\n" + ")"); @@ -484,19 +488,22 @@ private void testCreateTableLike() assertUpdate(session, "CREATE TABLE test_create_table_like_copy1 (LIKE test_create_table_like_original)"); assertEquals(getTablePropertiesString("test_create_table_like_copy1"), "WITH (\n" + - " format = 'PARQUET'\n" + + " format = 'PARQUET',\n" + + " format_version = '1'\n" + ")"); dropTable(session, "test_create_table_like_copy1"); assertUpdate(session, "CREATE TABLE test_create_table_like_copy2 (LIKE test_create_table_like_original EXCLUDING PROPERTIES)"); assertEquals(getTablePropertiesString("test_create_table_like_copy2"), "WITH (\n" + - " format = 'PARQUET'\n" + + " format = 'PARQUET',\n" + + " format_version = '1'\n" + ")"); dropTable(session, "test_create_table_like_copy2"); assertUpdate(session, "CREATE TABLE test_create_table_like_copy3 (LIKE test_create_table_like_original INCLUDING PROPERTIES)"); assertEquals(getTablePropertiesString("test_create_table_like_copy3"), "WITH (\n" + " format = 'PARQUET',\n" + + " format_version = '1',\n" + " partitioning = ARRAY['adate']\n" + ")"); dropTable(session, "test_create_table_like_copy3"); @@ -504,6 +511,7 @@ private void testCreateTableLike() assertUpdate(session, "CREATE TABLE test_create_table_like_copy4 (LIKE test_create_table_like_original INCLUDING PROPERTIES) WITH (format = 'ORC')"); assertEquals(getTablePropertiesString("test_create_table_like_copy4"), "WITH (\n" + " format = 'ORC',\n" + + " format_version = '1',\n" + " partitioning = ARRAY['adate']\n" + ")"); dropTable(session, "test_create_table_like_copy4"); @@ -511,6 +519,53 @@ private void testCreateTableLike() dropTable(session, "test_create_table_like_original"); } + @Test + public void testCreateTableWithFormatVersion() + { + testWithAllFormatVersions(this::testCreateTableWithFormatVersion); + } + + private void testCreateTableWithFormatVersion(Session session, String formatVersion) + { + @Language("SQL") String createTable = "" + + "CREATE TABLE test_create_table_with_format_version " + + "WITH (" + + "format = 'PARQUET', " + + "format_version = '" + formatVersion + "'" + + ") " + + "AS " + + "SELECT orderkey AS order_key, shippriority AS ship_priority, orderstatus AS order_status " + + "FROM tpch.tiny.orders"; + + assertUpdate(session, createTable, "SELECT count(*) from orders"); + + String createTableSql = format("" + + "CREATE TABLE %s.%s.%s (\n" + + " \"order_key\" bigint,\n" + + " \"ship_priority\" integer,\n" + + " \"order_status\" varchar\n" + + ")\n" + + "WITH (\n" + + " format = 'PARQUET',\n" + + " format_version = '%s'\n" + + ")", + getSession().getCatalog().get(), + getSession().getSchema().get(), + "test_create_table_with_format_version", + formatVersion); + + MaterializedResult actualResult = computeActual("SHOW CREATE TABLE test_create_table_with_format_version"); + assertEquals(getOnlyElement(actualResult.getOnlyColumnAsSet()), createTableSql); + + dropTable(session, "test_create_table_with_format_version"); + } + + private void testWithAllFormatVersions(BiConsumer test) + { + test.accept(getSession(), "1"); + test.accept(getSession(), "2"); + } + private String getTablePropertiesString(String tableName) { MaterializedResult showCreateTable = computeActual("SHOW CREATE TABLE " + tableName);