From 2b0077aaa7a5af6a52e38001102adce2befa51e4 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 7 Jun 2022 13:23:10 +0900 Subject: [PATCH 1/2] Extract utility class for Delta Lake product test --- ...akeDatabricksCreateTableCompatibility.java | 14 +------ .../deltalake/util/DeltaLakeTestUtils.java | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DeltaLakeTestUtils.java diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java index 1ecd9fee3c9d..01a81dce9db5 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java @@ -24,6 +24,8 @@ import static io.trino.tempto.assertions.QueryAssert.assertThat; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnDelta; +import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnTrino; import static io.trino.tests.product.hive.util.TemporaryHiveTable.randomTableSuffix; import static io.trino.tests.product.utils.QueryExecutors.onDelta; import static io.trino.tests.product.utils.QueryExecutors.onTrino; @@ -248,16 +250,4 @@ public void testCreateTableWithColumnCommentOnDelta() onDelta().executeQuery("DROP TABLE default." + tableName); } } - - private static String getColumnCommentOnTrino(String schemaName, String tableName, String columnName) - { - QueryResult result = onTrino().executeQuery("SELECT comment FROM information_schema.columns WHERE table_schema = '" + schemaName + "' AND table_name = '" + tableName + "' AND column_name = '" + columnName + "'"); - return (String) result.row(0).get(0); - } - - private static String getColumnCommentOnDelta(String schemaName, String tableName, String columnName) - { - QueryResult result = onDelta().executeQuery(format("DESCRIBE %s.%s %s", schemaName, tableName, columnName)); - return (String) result.row(2).get(1); - } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DeltaLakeTestUtils.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DeltaLakeTestUtils.java new file mode 100644 index 000000000000..a01cfcc5c916 --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DeltaLakeTestUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.deltalake.util; + +import io.trino.tempto.query.QueryResult; + +import static io.trino.tests.product.utils.QueryExecutors.onDelta; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.lang.String.format; + +public final class DeltaLakeTestUtils +{ + private DeltaLakeTestUtils() {} + + public static String getColumnCommentOnTrino(String schemaName, String tableName, String columnName) + { + QueryResult result = onTrino().executeQuery("SELECT comment FROM information_schema.columns WHERE table_schema = '" + schemaName + "' AND table_name = '" + tableName + "' AND column_name = '" + columnName + "'"); + return (String) result.row(0).get(0); + } + + public static String getColumnCommentOnDelta(String schemaName, String tableName, String columnName) + { + QueryResult result = onDelta().executeQuery(format("DESCRIBE %s.%s %s", schemaName, tableName, columnName)); + return (String) result.row(2).get(1); + } +} From 9f10cc0e78a0c35bcd3ac256bb9ff489ae18ee88 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 7 Jun 2022 13:24:27 +0900 Subject: [PATCH 2/2] Support comment when adding a new column in Delta Lake --- .../plugin/deltalake/DeltaLakeMetadata.java | 11 ++-- .../BaseDeltaLakeMinioConnectorTest.java | 1 - .../TestDeltaLakeAlterTableCompatibility.java | 51 +++++++++++++++++++ 3 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java index 7dc1e882cc7b..ccc516275952 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java @@ -944,10 +944,6 @@ private static boolean isCreatedBy(Table table, String queryId) @Override public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata newColumnMetadata) { - if (newColumnMetadata.getComment() != null) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support adding columns with comments"); - } - DeltaLakeTableHandle handle = (DeltaLakeTableHandle) tableHandle; ConnectorTableMetadata tableMetadata = getTableMetadata(session, handle); @@ -961,6 +957,11 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle .map(column -> toColumnHandle(column, partitionColumns)) .collect(toImmutableList())); columnsBuilder.add(toColumnHandle(newColumnMetadata, partitionColumns)); + ImmutableMap.Builder columnComments = ImmutableMap.builder(); + columnComments.putAll(getColumnComments(handle.getMetadataEntry())); + if (newColumnMetadata.getComment() != null) { + columnComments.put(newColumnMetadata.getName(), newColumnMetadata.getComment()); + } Optional checkpointInterval = DeltaLakeTableProperties.getCheckpointInterval(tableMetadata.getProperties()); @@ -971,7 +972,7 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle handle.getMetadataEntry().getId(), columnsBuilder.build(), partitionColumns, - getColumnComments(handle.getMetadataEntry()), + columnComments.buildOrThrow(), buildDeltaMetadataConfiguration(checkpointInterval), ADD_COLUMN_OPERATION, session, diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java index dd419ffcd7ff..e5066ea5365a 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java @@ -106,7 +106,6 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) case SUPPORTS_TOPN_PUSHDOWN: case SUPPORTS_AGGREGATION_PUSHDOWN: case SUPPORTS_RENAME_TABLE: - case SUPPORTS_ADD_COLUMN_WITH_COMMENT: case SUPPORTS_DROP_COLUMN: case SUPPORTS_RENAME_COLUMN: case SUPPORTS_COMMENT_ON_TABLE: diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java new file mode 100644 index 000000000000..abd118d4939f --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.deltalake; + +import org.testng.annotations.Test; + +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnDelta; +import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnTrino; +import static io.trino.tests.product.hive.util.TemporaryHiveTable.randomTableSuffix; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; + +public class TestDeltaLakeAlterTableCompatibility + extends BaseTestDeltaLakeS3Storage +{ + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + public void testAddColumnWithCommentOnTrino() + { + String tableName = "test_dl_add_column_with_comment_" + randomTableSuffix(); + String tableDirectory = "databricks-compatibility-test-" + tableName; + + onTrino().executeQuery(format("CREATE TABLE delta.default.%s (col INT) WITH (location = 's3://%s/%s')", + tableName, + bucketName, + tableDirectory)); + + try { + onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " ADD COLUMN new_col INT COMMENT 'new column comment'"); + assertEquals(getColumnCommentOnTrino("default", tableName, "new_col"), "new column comment"); + assertEquals(getColumnCommentOnDelta("default", tableName, "new_col"), "new column comment"); + } + finally { + onTrino().executeQuery("DROP TABLE delta.default." + tableName); + } + } +}