From f2cb348a385ad8be26c165b071d50235272f8e42 Mon Sep 17 00:00:00 2001 From: polaris Date: Sat, 28 May 2022 15:38:46 +0800 Subject: [PATCH 1/2] Extract `getIcebergTableProperties` method in `IcebergUtil` --- .../trino/plugin/iceberg/IcebergMetadata.java | 29 ++------------- .../io/trino/plugin/iceberg/IcebergUtil.java | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index c2ce359fc14a..b27b6d0ba092 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -169,15 +169,13 @@ import static io.trino.plugin.iceberg.IcebergSessionProperties.isStatisticsEnabled; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; -import static io.trino.plugin.iceberg.IcebergTableProperties.LOCATION_PROPERTY; -import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_COLUMNS; -import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_FPP; import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.getPartitioning; import static io.trino.plugin.iceberg.IcebergUtil.deserializePartitionValue; import static io.trino.plugin.iceberg.IcebergUtil.getColumnHandle; import static io.trino.plugin.iceberg.IcebergUtil.getColumns; import static io.trino.plugin.iceberg.IcebergUtil.getFileFormat; +import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableProperties; import static io.trino.plugin.iceberg.IcebergUtil.getPartitionKeys; import static io.trino.plugin.iceberg.IcebergUtil.getTableComment; import static io.trino.plugin.iceberg.IcebergUtil.newCreateTableTransaction; @@ -1287,30 +1285,7 @@ private ConnectorTableMetadata getTableMetadata(ConnectorSession session, Schema columns.addAll(getColumnMetadatas(icebergTable)); columns.add(pathColumnMetadata()); - ImmutableMap.Builder properties = ImmutableMap.builder(); - properties.put(FILE_FORMAT_PROPERTY, getFileFormat(icebergTable)); - if (!icebergTable.spec().fields().isEmpty()) { - properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec())); - } - - if (!icebergTable.location().isEmpty()) { - properties.put(LOCATION_PROPERTY, icebergTable.location()); - } - - int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); - properties.put(FORMAT_VERSION_PROPERTY, formatVersion); - - // iceberg ORC format bloom filter properties - String orcBloomFilterColumns = icebergTable.properties().get(ORC_BLOOM_FILTER_COLUMNS_KEY); - if (orcBloomFilterColumns != null) { - properties.put(ORC_BLOOM_FILTER_COLUMNS, Splitter.on(',').trimResults().omitEmptyStrings().splitToList(orcBloomFilterColumns)); - } - String orcBloomFilterFpp = icebergTable.properties().get(ORC_BLOOM_FILTER_FPP_KEY); - if (orcBloomFilterFpp != null) { - properties.put(ORC_BLOOM_FILTER_FPP, Double.parseDouble(orcBloomFilterFpp)); - } - - return new ConnectorTableMetadata(table, columns.build(), properties.buildOrThrow(), getTableComment(icebergTable)); + return new ConnectorTableMetadata(table, columns.build(), getIcebergTableProperties(icebergTable), getTableComment(icebergTable)); } private List getColumnMetadatas(Table table) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java index d69f4f57b1ab..42b462edc66a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java @@ -14,6 +14,7 @@ package io.trino.plugin.iceberg; import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slice; @@ -77,12 +78,18 @@ import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_INVALID_SNAPSHOT_ID; import static io.trino.plugin.iceberg.IcebergMetadata.ORC_BLOOM_FILTER_COLUMNS_KEY; import static io.trino.plugin.iceberg.IcebergMetadata.ORC_BLOOM_FILTER_FPP_KEY; +import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; +import static io.trino.plugin.iceberg.IcebergTableProperties.LOCATION_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_COLUMNS; +import static io.trino.plugin.iceberg.IcebergTableProperties.ORC_BLOOM_FILTER_FPP; +import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.getOrcBloomFilterColumns; import static io.trino.plugin.iceberg.IcebergTableProperties.getOrcBloomFilterFpp; import static io.trino.plugin.iceberg.IcebergTableProperties.getPartitioning; import static io.trino.plugin.iceberg.IcebergTableProperties.getTableLocation; import static io.trino.plugin.iceberg.PartitionFields.parsePartitionFields; +import static io.trino.plugin.iceberg.PartitionFields.toPartitionFields; import static io.trino.plugin.iceberg.TypeConverter.toIcebergType; import static io.trino.plugin.iceberg.TypeConverter.toTrinoType; import static io.trino.plugin.iceberg.util.Timestamps.timestampTzFromMicros; @@ -162,6 +169,34 @@ public static Table getIcebergTableWithMetadata( return new BaseTable(operations, quotedTableName(table)); } + public static Map getIcebergTableProperties(Table icebergTable) + { + ImmutableMap.Builder properties = ImmutableMap.builder(); + properties.put(FILE_FORMAT_PROPERTY, getFileFormat(icebergTable)); + if (!icebergTable.spec().fields().isEmpty()) { + properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec())); + } + + if (!icebergTable.location().isEmpty()) { + properties.put(LOCATION_PROPERTY, icebergTable.location()); + } + + int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + properties.put(FORMAT_VERSION_PROPERTY, formatVersion); + + // iceberg ORC format bloom filter properties + String orcBloomFilterColumns = icebergTable.properties().get(ORC_BLOOM_FILTER_COLUMNS_KEY); + if (orcBloomFilterColumns != null) { + properties.put(ORC_BLOOM_FILTER_COLUMNS, Splitter.on(',').trimResults().omitEmptyStrings().splitToList(orcBloomFilterColumns)); + } + String orcBloomFilterFpp = icebergTable.properties().get(ORC_BLOOM_FILTER_FPP_KEY); + if (orcBloomFilterFpp != null) { + properties.put(ORC_BLOOM_FILTER_FPP, Double.parseDouble(orcBloomFilterFpp)); + } + + return properties.buildOrThrow(); + } + public static long resolveSnapshotId(Table table, long snapshotId) { if (table.snapshot(snapshotId) != null) { From 8cca94bc2866874093d6faa1d2a43d83d7a78929 Mon Sep 17 00:00:00 2001 From: polaris Date: Sat, 28 May 2022 16:06:57 +0800 Subject: [PATCH 2/2] Add Iceberg properties in `SHOW CREATE MATERIALIZED VIEW` --- .../iceberg/catalog/AbstractTrinoCatalog.java | 11 ++------ .../BaseIcebergMaterializedViewTest.java | 25 ++++++++++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java index a890c5e8b608..eb593733530a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java @@ -62,8 +62,7 @@ import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; import static io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.decodeMaterializedViewData; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; -import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; -import static io.trino.plugin.iceberg.PartitionFields.toPartitionFields; +import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableProperties; import static io.trino.spi.StandardErrorCode.TABLE_NOT_FOUND; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -282,12 +281,6 @@ protected ConnectorMaterializedViewDefinition getMaterializedViewDefinition( String viewOriginalText, String storageTableName) { - ImmutableMap.Builder properties = ImmutableMap.builder(); - properties.put(FILE_FORMAT_PROPERTY, IcebergUtil.getFileFormat(icebergTable)); - if (!icebergTable.spec().fields().isEmpty()) { - properties.put(PARTITIONING_PROPERTY, toPartitionFields(icebergTable.spec())); - } - IcebergMaterializedViewDefinition definition = decodeMaterializedViewData(viewOriginalText); return new ConnectorMaterializedViewDefinition( definition.getOriginalSql(), @@ -299,7 +292,7 @@ protected ConnectorMaterializedViewDefinition getMaterializedViewDefinition( .collect(toImmutableList()), definition.getComment(), owner, - properties.buildOrThrow()); + getIcebergTableProperties(icebergTable)); } protected Map createMaterializedViewProperties(ConnectorSession session, String storageTableName) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java index 280364e264d2..a220a77c8ad4 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java @@ -28,6 +28,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.nio.file.Path; import java.util.Optional; import java.util.Set; @@ -151,14 +152,23 @@ public void testCreateWithDuplicateSourceTableSucceeds() public void testShowCreate() { assertUpdate("CREATE MATERIALIZED VIEW materialized_view_with_property " + - "WITH (partitioning = ARRAY['_date']) AS " + + "WITH (\n" + + " partitioning = ARRAY['_date'],\n" + + " orc_bloom_filter_columns = ARRAY['_date'],\n" + + " orc_bloom_filter_fpp = 0.1) AS " + "SELECT _bigint, _date FROM base_table1"); assertQuery("SELECT COUNT(*) FROM materialized_view_with_property", "VALUES 6"); - assertThat(computeActual("SHOW CREATE MATERIALIZED VIEW materialized_view_with_property").getOnlyValue()) - .isEqualTo( - "CREATE MATERIALIZED VIEW iceberg." + getSchemaName() + ".materialized_view_with_property\n" + + + Path schemaDirectory = getDistributedQueryRunner().getCoordinator().getBaseDataDir().resolve("iceberg_data/tpch"); + assertThat((String) computeScalar("SHOW CREATE MATERIALIZED VIEW materialized_view_with_property")) + .matches( + "\\QCREATE MATERIALIZED VIEW iceberg." + getSchemaName() + ".materialized_view_with_property\n" + "WITH (\n" + " format = 'ORC',\n" + + " format_version = 2,\n" + + " location = '" + schemaDirectory + "/st_\\E[0-9a-f]+\\Q',\n" + + " orc_bloom_filter_columns = ARRAY['_date'],\n" + + " orc_bloom_filter_fpp = 1E-1,\n" + " partitioning = ARRAY['_date']\n" + ") AS\n" + "SELECT\n" + @@ -425,10 +435,13 @@ public void testSqlFeatures() assertQueryFails("show create view materialized_view_window", "line 1:1: Relation '" + qualifiedMaterializedViewName + "' is a materialized view, not a view"); - assertThat(computeScalar("show create materialized view materialized_view_window")) - .isEqualTo("CREATE MATERIALIZED VIEW " + qualifiedMaterializedViewName + "\n" + + Path schemaDirectory = getDistributedQueryRunner().getCoordinator().getBaseDataDir().resolve("iceberg_data/tpch"); + assertThat((String) computeScalar("show create materialized view materialized_view_window")) + .matches("\\QCREATE MATERIALIZED VIEW " + qualifiedMaterializedViewName + "\n" + "WITH (\n" + " format = 'ORC',\n" + + " format_version = 2,\n" + + " location = '" + schemaDirectory + "/st_\\E[0-9a-f]+\\Q',\n" + " partitioning = ARRAY['_date']\n" + ") AS\n" + "SELECT\n" +