diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java index 435e099fb41c..6259773382b1 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java @@ -21,6 +21,8 @@ import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; +import java.util.Optional; + import static io.trino.plugin.hive.HiveCompressionCodec.ZSTD; import static io.trino.plugin.iceberg.CatalogType.HIVE_METASTORE; import static io.trino.plugin.iceberg.IcebergFileFormat.ORC; @@ -37,6 +39,7 @@ public class IcebergConfig private Duration dynamicFilteringWaitTimeout = new Duration(0, SECONDS); private boolean tableStatisticsEnabled = true; private boolean projectionPushdownEnabled = true; + private Optional hiveCatalogName = Optional.empty(); public CatalogType getCatalogType() { @@ -166,4 +169,17 @@ public IcebergConfig setProjectionPushdownEnabled(boolean projectionPushdownEnab this.projectionPushdownEnabled = projectionPushdownEnabled; return this; } + + public Optional getHiveCatalogName() + { + return hiveCatalogName; + } + + @Config("iceberg.hive-catalog-name") + @ConfigDescription("Catalog to redirect to when a Hive table is referenced") + public IcebergConfig setHiveCatalogName(String hiveCatalogName) + { + this.hiveCatalogName = Optional.ofNullable(hiveCatalogName); + return this; + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 3f30e98876d6..2f88c51872ab 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -63,6 +63,7 @@ import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.SchemaTablePrefix; import io.trino.spi.connector.SystemTable; +import io.trino.spi.connector.TableColumnsMetadata; import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.expression.ConnectorExpression; import io.trino.spi.expression.Variable; @@ -115,6 +116,7 @@ import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -151,7 +153,6 @@ import static io.trino.spi.connector.RetryMode.NO_RETRIES; import static io.trino.spi.type.BigintType.BIGINT; import static java.lang.String.format; -import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; import static java.util.stream.Collectors.joining; @@ -206,7 +207,10 @@ public Optional getSchemaOwner(ConnectorSession session, Catalog public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { IcebergTableName name = IcebergTableName.from(tableName.getTableName()); - verify(name.getTableType() == DATA, "Wrong table type: " + name.getTableNameWithType()); + if (name.getTableType() != DATA) { + // Pretend the table does not exist to produce better error message in case of table redirects to Hive + return null; + } Table table; try { @@ -236,6 +240,7 @@ public Optional getSystemTable(ConnectorSession session, SchemaTabl .map(systemTable -> new ClassLoaderSafeSystemTable(systemTable, getClass().getClassLoader())); } + @SuppressWarnings("TryWithIdenticalCatches") private Optional getRawSystemTable(ConnectorSession session, SchemaTableName tableName) { IcebergTableName name = IcebergTableName.from(tableName.getTableName()); @@ -251,6 +256,10 @@ private Optional getRawSystemTable(ConnectorSession session, Schema catch (TableNotFoundException e) { return Optional.empty(); } + catch (UnknownTableTypeException e) { + // avoid dealing with non Iceberg tables + return Optional.empty(); + } SchemaTableName systemTableName = new SchemaTableName(tableName.getSchemaName(), name.getTableNameWithType()); switch (name.getTableType()) { @@ -395,27 +404,43 @@ public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTable @Override public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) { - List tables = prefix.getTable() - .map(ignored -> singletonList(prefix.toSchemaTableName())) - .orElseGet(() -> listTables(session, prefix.getSchema())); + throw new UnsupportedOperationException("The deprecated listTableColumns is not supported because streamTableColumns is implemented instead"); + } - ImmutableMap.Builder> columns = ImmutableMap.builder(); - for (SchemaTableName table : tables) { - try { - columns.put(table, getTableMetadata(session, table).getColumns()); - } - catch (TableNotFoundException e) { - // table disappeared during listing operation - } - catch (UnknownTableTypeException e) { - // ignore table of unknown type - } - catch (RuntimeException e) { - // Table can be being removed and this may cause all sorts of exceptions. Log, because we're catching broadly. - log.warn(e, "Failed to access metadata of table %s during column listing for %s", table, prefix); - } + @Override + @SuppressWarnings("TryWithIdenticalCatches") + public Stream streamTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + requireNonNull(prefix, "prefix is null"); + List schemaTableNames; + if (prefix.getTable().isEmpty()) { + schemaTableNames = catalog.listTables(session, prefix.getSchema()); + } + else { + schemaTableNames = ImmutableList.of(prefix.toSchemaTableName()); } - return columns.buildOrThrow(); + return schemaTableNames.stream() + .flatMap(tableName -> { + try { + if (redirectTable(session, tableName).isPresent()) { + return Stream.of(TableColumnsMetadata.forRedirectedTable(tableName)); + } + return Stream.of(TableColumnsMetadata.forTable(tableName, getTableMetadata(session, tableName).getColumns())); + } + catch (TableNotFoundException e) { + // Table disappeared during listing operation + return Stream.empty(); + } + catch (UnknownTableTypeException e) { + // Skip unsupported table type in case that the table redirects are not enabled + return Stream.empty(); + } + catch (RuntimeException e) { + // Table can be being removed and this may cause all sorts of exceptions. Log, because we're catching broadly. + log.warn(e, "Failed to access metadata of table %s during streaming table columns for %s", tableName, prefix); + return Stream.empty(); + } + }); } @Override @@ -1394,6 +1419,12 @@ private Map> getMaterializedViewToken(ConnectorSess return viewToken; } + @Override + public Optional redirectTable(ConnectorSession session, SchemaTableName tableName) + { + return catalog.redirectTable(session, tableName); + } + private static class TableToken { // Current Snapshot ID of the table diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java index 0d6bd977836d..c5326aeaa418 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import java.util.List; +import java.util.Optional; import java.util.concurrent.ThreadLocalRandom; import static com.google.common.base.Preconditions.checkArgument; @@ -41,6 +42,7 @@ import static io.trino.spi.session.PropertyMetadata.doubleProperty; import static io.trino.spi.session.PropertyMetadata.enumProperty; import static io.trino.spi.session.PropertyMetadata.integerProperty; +import static io.trino.spi.session.PropertyMetadata.stringProperty; import static java.lang.String.format; public final class IcebergSessionProperties @@ -71,6 +73,7 @@ public final class IcebergSessionProperties private static final String STATISTICS_ENABLED = "statistics_enabled"; private static final String PROJECTION_PUSHDOWN_ENABLED = "projection_pushdown_enabled"; private static final String TARGET_MAX_FILE_SIZE = "target_max_file_size"; + private static final String HIVE_CATALOG_NAME = "hive_catalog_name"; private final List> sessionProperties; @@ -219,6 +222,13 @@ public IcebergSessionProperties( "Target maximum size of written files; the actual size may be larger", hiveConfig.getTargetMaxFileSize(), false)) + .add(stringProperty( + HIVE_CATALOG_NAME, + "Catalog to redirect to when a Hive table is referenced", + icebergConfig.getHiveCatalogName().orElse(null), + // Session-level redirections configuration does not work well with views, as view body is analyzed in context + // of a session with properties stripped off. Thus, this property is more of a test-only, or at most POC usefulness. + true)) .build(); } @@ -359,4 +369,9 @@ public static long getTargetMaxFileSize(ConnectorSession session) { return session.getProperty(TARGET_MAX_FILE_SIZE, DataSize.class).toBytes(); } + + public static Optional getHiveCatalogName(ConnectorSession session) + { + return Optional.ofNullable(session.getProperty(HIVE_CATALOG_NAME, String.class)); + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java index 514ab45de430..805e1d3d1bb5 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java @@ -15,6 +15,7 @@ import io.trino.plugin.iceberg.ColumnIdentity; import io.trino.plugin.iceberg.UnknownTableTypeException; +import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorViewDefinition; @@ -119,4 +120,6 @@ void createMaterializedView( void renameMaterializedView(ConnectorSession session, SchemaTableName source, SchemaTableName target); void updateColumnComment(ConnectorSession session, SchemaTableName schemaTableName, ColumnIdentity columnIdentity, Optional comment); + + Optional redirectTable(ConnectorSession session, SchemaTableName tableName); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 0a1b26ebf8ed..0d8da9c2ee5c 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -40,6 +40,7 @@ import io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; import io.trino.spi.TrinoException; +import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorViewDefinition; @@ -74,16 +75,22 @@ import static io.trino.plugin.hive.ViewReaderUtil.encodeViewData; import static io.trino.plugin.hive.ViewReaderUtil.isPrestoView; import static io.trino.plugin.hive.metastore.glue.AwsSdkUtil.getPaginatedResults; +import static io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_CATALOG_ERROR; import static io.trino.plugin.iceberg.IcebergSchemaProperties.LOCATION_PROPERTY; +import static io.trino.plugin.iceberg.IcebergSessionProperties.getHiveCatalogName; import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata; import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName; import static io.trino.plugin.iceberg.IcebergUtil.validateTableCanBeDropped; import static io.trino.plugin.iceberg.catalog.glue.GlueIcebergUtil.getTableInput; import static io.trino.plugin.iceberg.catalog.glue.GlueIcebergUtil.getViewTableInput; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.connector.SchemaTableName.schemaTableName; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW; +import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE; +import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP; import static org.apache.iceberg.CatalogUtil.dropTableData; public class TrinoGlueCatalog @@ -575,4 +582,40 @@ public void renameMaterializedView(ConnectorSession session, SchemaTableName sou { throw new TrinoException(NOT_SUPPORTED, "renameMaterializedView is not supported for Iceberg Glue catalogs"); } + + @Override + public Optional redirectTable(ConnectorSession session, SchemaTableName tableName) + { + requireNonNull(session, "session is null"); + requireNonNull(tableName, "tableName is null"); + Optional targetCatalogName = getHiveCatalogName(session); + if (targetCatalogName.isEmpty()) { + return Optional.empty(); + } + if (isHiveSystemSchema(tableName.getSchemaName())) { + return Optional.empty(); + } + + // we need to chop off any "$partitions" and similar suffixes from table name while querying the metastore for the Table object + int metadataMarkerIndex = tableName.getTableName().lastIndexOf('$'); + SchemaTableName tableNameBase = (metadataMarkerIndex == -1) ? tableName : schemaTableName( + tableName.getSchemaName(), + tableName.getTableName().substring(0, metadataMarkerIndex)); + + Optional table = getTable(new SchemaTableName(tableNameBase.getSchemaName(), tableNameBase.getTableName())); + + if (table.isEmpty() || VIRTUAL_VIEW.name().equals(table.get().getTableType())) { + return Optional.empty(); + } + if (!isIcebergTable(table.get())) { + // After redirecting, use the original table name, with "$partitions" and similar suffixes + return targetCatalogName.map(catalog -> new CatalogSchemaTableName(catalog, tableName)); + } + return Optional.empty(); + } + + private static boolean isIcebergTable(com.amazonaws.services.glue.model.Table table) + { + return ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getParameters().get(TABLE_TYPE_PROP)); + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java index 3ade57c1d6d3..1fa156896c47 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java @@ -91,9 +91,11 @@ import static io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.encodeMaterializedViewData; import static io.trino.plugin.iceberg.IcebergMaterializedViewDefinition.fromConnectorMaterializedViewDefinition; import static io.trino.plugin.iceberg.IcebergSchemaProperties.getSchemaLocation; +import static io.trino.plugin.iceberg.IcebergSessionProperties.getHiveCatalogName; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.PARTITIONING_PROPERTY; import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata; +import static io.trino.plugin.iceberg.IcebergUtil.isIcebergTable; import static io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable; import static io.trino.plugin.iceberg.IcebergUtil.validateTableCanBeDropped; import static io.trino.plugin.iceberg.PartitionFields.toPartitionFields; @@ -101,6 +103,7 @@ import static io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.StandardErrorCode.SCHEMA_NOT_EMPTY; +import static io.trino.spi.connector.SchemaTableName.schemaTableName; import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; import static org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW; @@ -622,6 +625,37 @@ private List listNamespaces(ConnectorSession session, Optional n return listNamespaces(session); } + @Override + public Optional redirectTable(ConnectorSession session, SchemaTableName tableName) + { + requireNonNull(session, "session is null"); + requireNonNull(tableName, "tableName is null"); + Optional targetCatalogName = getHiveCatalogName(session); + if (targetCatalogName.isEmpty()) { + return Optional.empty(); + } + if (isHiveSystemSchema(tableName.getSchemaName())) { + return Optional.empty(); + } + + // we need to chop off any "$partitions" and similar suffixes from table name while querying the metastore for the Table object + int metadataMarkerIndex = tableName.getTableName().lastIndexOf('$'); + SchemaTableName tableNameBase = (metadataMarkerIndex == -1) ? tableName : schemaTableName( + tableName.getSchemaName(), + tableName.getTableName().substring(0, metadataMarkerIndex)); + + Optional table = metastore.getTable(tableNameBase.getSchemaName(), tableNameBase.getTableName()); + + if (table.isEmpty() || isHiveOrPrestoView(table.get().getTableType())) { + return Optional.empty(); + } + if (!isIcebergTable(table.get())) { + // After redirecting, use the original table name, with "$partitions" and similar suffixes + return targetCatalogName.map(catalog -> new CatalogSchemaTableName(catalog, tableName)); + } + return Optional.empty(); + } + private static class MaterializedViewMayBeBeingRemovedException extends RuntimeException { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java new file mode 100644 index 000000000000..a5aad0c7c525 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseSharedMetastoreTest.java @@ -0,0 +1,134 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import io.trino.testing.AbstractTestQueryFramework; +import org.testng.annotations.Test; + +import java.nio.file.Path; + +import static io.trino.testing.sql.TestTable.randomTableSuffix; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testng.Assert.assertEquals; + +public abstract class BaseSharedMetastoreTest + extends AbstractTestQueryFramework +{ + protected final String schema = "test_shared_schema_" + randomTableSuffix(); + protected Path dataDirectory; + + protected abstract String getExpectedHiveCreateSchema(String catalogName); + + protected abstract String getExpectedIcebergCreateSchema(String catalogName); + + @Test + public void testSelect() + { + assertQuery("SELECT * FROM iceberg." + schema + ".nation", "SELECT * FROM nation"); + assertQuery("SELECT * FROM hive." + schema + ".region", "SELECT * FROM region"); + assertQuery("SELECT * FROM hive_with_redirections." + schema + ".nation", "SELECT * FROM nation"); + assertQuery("SELECT * FROM hive_with_redirections." + schema + ".region", "SELECT * FROM region"); + assertQuery("SELECT * FROM iceberg_with_redirections." + schema + ".nation", "SELECT * FROM nation"); + assertQuery("SELECT * FROM iceberg_with_redirections." + schema + ".region", "SELECT * FROM region"); + + assertThatThrownBy(() -> query("SELECT * FROM iceberg." + schema + ".region")) + .hasMessageContaining("Not an Iceberg table"); + assertThatThrownBy(() -> query("SELECT * FROM hive." + schema + ".nation")) + .hasMessageContaining("Cannot query Iceberg table"); + } + + @Test + public void testReadInformationSchema() + { + assertThat(query("SELECT table_schema FROM hive.information_schema.tables WHERE table_name = 'region'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SELECT table_schema FROM iceberg.information_schema.tables WHERE table_name = 'nation'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SELECT table_schema FROM hive_with_redirections.information_schema.tables WHERE table_name = 'region'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SELECT table_schema FROM hive_with_redirections.information_schema.tables WHERE table_name = 'nation'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SELECT table_schema FROM iceberg_with_redirections.information_schema.tables WHERE table_name = 'region'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + + assertQuery("SELECT table_name, column_name from hive.information_schema.columns WHERE table_schema = '" + schema + "'", + "VALUES ('region', 'regionkey'), ('region', 'name'), ('region', 'comment')"); + assertQuery("SELECT table_name, column_name from iceberg.information_schema.columns WHERE table_schema = '" + schema + "'", + "VALUES ('nation', 'nationkey'), ('nation', 'name'), ('nation', 'regionkey'), ('nation', 'comment')"); + assertQuery("SELECT table_name, column_name from hive_with_redirections.information_schema.columns WHERE table_schema = '" + schema + "'", + "VALUES" + + "('region', 'regionkey'), ('region', 'name'), ('region', 'comment'), " + + "('nation', 'nationkey'), ('nation', 'name'), ('nation', 'regionkey'), ('nation', 'comment')"); + assertQuery("SELECT table_name, column_name from iceberg_with_redirections.information_schema.columns WHERE table_schema = '" + schema + "'", + "VALUES" + + "('region', 'regionkey'), ('region', 'name'), ('region', 'comment'), " + + "('nation', 'nationkey'), ('nation', 'name'), ('nation', 'regionkey'), ('nation', 'comment')"); + } + + @Test + public void testShowTables() + { + assertQuery("SHOW TABLES FROM iceberg." + schema, "VALUES 'region', 'nation'"); + assertQuery("SHOW TABLES FROM hive." + schema, "VALUES 'region', 'nation'"); + assertQuery("SHOW TABLES FROM hive_with_redirections." + schema, "VALUES 'region', 'nation'"); + assertQuery("SHOW TABLES FROM iceberg_with_redirections." + schema, "VALUES 'region', 'nation'"); + + assertThatThrownBy(() -> query("SHOW CREATE TABLE iceberg." + schema + ".region")) + .hasMessageContaining("Not an Iceberg table"); + assertThatThrownBy(() -> query("SHOW CREATE TABLE hive." + schema + ".nation")) + .hasMessageContaining("Cannot query Iceberg table"); + + assertThatThrownBy(() -> query("DESCRIBE iceberg." + schema + ".region")) + .hasMessageContaining("Not an Iceberg table"); + assertThatThrownBy(() -> query("DESCRIBE hive." + schema + ".nation")) + .hasMessageContaining("Cannot query Iceberg table"); + } + + @Test + public void testShowSchemas() + { + assertThat(query("SHOW SCHEMAS FROM hive")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SHOW SCHEMAS FROM iceberg")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SHOW SCHEMAS FROM hive_with_redirections")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + + String showCreateHiveSchema = (String) computeActual("SHOW CREATE SCHEMA hive." + schema).getOnlyValue(); + assertEquals( + showCreateHiveSchema, + getExpectedHiveCreateSchema("hive")); + String showCreateIcebergSchema = (String) computeActual("SHOW CREATE SCHEMA iceberg." + schema).getOnlyValue(); + assertEquals( + showCreateIcebergSchema, + getExpectedIcebergCreateSchema("iceberg")); + String showCreateHiveWithRedirectionsSchema = (String) computeActual("SHOW CREATE SCHEMA hive_with_redirections." + schema).getOnlyValue(); + assertEquals( + showCreateHiveWithRedirectionsSchema, + getExpectedHiveCreateSchema("hive_with_redirections")); + String showCreateIcebergWithRedirectionsSchema = (String) computeActual("SHOW CREATE SCHEMA iceberg_with_redirections." + schema).getOnlyValue(); + assertEquals( + showCreateIcebergWithRedirectionsSchema, + getExpectedIcebergCreateSchema("iceberg_with_redirections")); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java index e57bdcd7e04d..b500262b3713 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java @@ -44,7 +44,8 @@ public void testDefaults() .setCatalogType(HIVE_METASTORE) .setDynamicFilteringWaitTimeout(new Duration(0, MINUTES)) .setTableStatisticsEnabled(true) - .setProjectionPushdownEnabled(true)); + .setProjectionPushdownEnabled(true) + .setHiveCatalogName(null)); } @Test @@ -60,6 +61,7 @@ public void testExplicitPropertyMappings() .put("iceberg.dynamic-filtering.wait-timeout", "1h") .put("iceberg.table-statistics-enabled", "false") .put("iceberg.projection-pushdown-enabled", "false") + .put("iceberg.hive-catalog-name", "hive") .buildOrThrow(); IcebergConfig expected = new IcebergConfig() @@ -71,7 +73,8 @@ public void testExplicitPropertyMappings() .setCatalogType(GLUE) .setDynamicFilteringWaitTimeout(Duration.valueOf("1h")) .setTableStatisticsEnabled(false) - .setProjectionPushdownEnabled(false); + .setProjectionPushdownEnabled(false) + .setHiveCatalogName("hive"); assertFullMapping(properties, expected); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveMetastore.java new file mode 100644 index 000000000000..b72dab0f4394 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestSharedHiveMetastore.java @@ -0,0 +1,140 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.trino.Session; +import io.trino.plugin.hive.HdfsConfig; +import io.trino.plugin.hive.HdfsConfigurationInitializer; +import io.trino.plugin.hive.HdfsEnvironment; +import io.trino.plugin.hive.HiveHdfsConfiguration; +import io.trino.plugin.hive.NodeVersion; +import io.trino.plugin.hive.TestingHivePlugin; +import io.trino.plugin.hive.authentication.NoHdfsAuthentication; +import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.MetastoreConfig; +import io.trino.plugin.hive.metastore.file.FileHiveMetastore; +import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig; +import io.trino.plugin.tpch.TpchPlugin; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import io.trino.tpch.TpchTable; +import org.testng.annotations.AfterClass; + +import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; +import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.trino.testing.QueryAssertions.copyTpchTables; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +public class TestSharedHiveMetastore + extends BaseSharedMetastoreTest +{ + private static final String HIVE_CATALOG = "hive"; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + Session icebergSession = testSessionBuilder() + .setCatalog(ICEBERG_CATALOG) + .setSchema(schema) + .build(); + Session hiveSession = testSessionBuilder() + .setCatalog(HIVE_CATALOG) + .setSchema(schema) + .build(); + + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(icebergSession).build(); + + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + + this.dataDirectory = queryRunner.getCoordinator().getBaseDataDir().resolve("iceberg_data"); + this.dataDirectory.toFile().deleteOnExit(); + + queryRunner.installPlugin(new IcebergPlugin()); + queryRunner.createCatalog( + ICEBERG_CATALOG, + "iceberg", + ImmutableMap.of( + "iceberg.catalog.type", "TESTING_FILE_METASTORE", + "hive.metastore.catalog.dir", dataDirectory.toString())); + queryRunner.createCatalog( + "iceberg_with_redirections", + "iceberg", + ImmutableMap.of( + "iceberg.catalog.type", "TESTING_FILE_METASTORE", + "hive.metastore.catalog.dir", dataDirectory.toString(), + "iceberg.hive-catalog-name", "hive")); + + HdfsConfig hdfsConfig = new HdfsConfig(); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment( + new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hdfsConfig), ImmutableSet.of()), + hdfsConfig, + new NoHdfsAuthentication()); + HiveMetastore metastore = new FileHiveMetastore( + new NodeVersion("testversion"), + hdfsEnvironment, + new MetastoreConfig(), + new FileHiveMetastoreConfig() + .setCatalogDirectory(dataDirectory.toFile().toURI().toString()) + .setMetastoreUser("test")); + queryRunner.installPlugin(new TestingHivePlugin(metastore)); + queryRunner.createCatalog(HIVE_CATALOG, "hive", ImmutableMap.of("hive.allow-drop-table", "true")); + queryRunner.createCatalog( + "hive_with_redirections", + "hive", + ImmutableMap.of("hive.iceberg-catalog-name", "iceberg")); + + queryRunner.execute("CREATE SCHEMA " + schema); + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, icebergSession, ImmutableList.of(TpchTable.NATION)); + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, hiveSession, ImmutableList.of(TpchTable.REGION)); + + return queryRunner; + } + + @AfterClass(alwaysRun = true) + public void cleanup() + { + assertQuerySucceeds("DROP TABLE IF EXISTS hive." + schema + ".region"); + assertQuerySucceeds("DROP TABLE IF EXISTS iceberg." + schema + ".nation"); + assertQuerySucceeds("DROP SCHEMA IF EXISTS hive." + schema); + } + + @Override + protected String getExpectedHiveCreateSchema(String catalogName) + { + String expectedHiveCreateSchema = "CREATE SCHEMA %s.%s\n" + + "AUTHORIZATION USER user\n" + + "WITH (\n" + + " location = 'file:%s/%s'\n" + + ")"; + + return format(expectedHiveCreateSchema, catalogName, schema, dataDirectory, schema); + } + + @Override + protected String getExpectedIcebergCreateSchema(String catalogName) + { + String expectedIcebergCreateSchema = "CREATE SCHEMA %s.%s\n" + + "AUTHORIZATION USER user\n" + + "WITH (\n" + + " location = '%s/%s'\n" + + ")"; + return format(expectedIcebergCreateSchema, catalogName, schema, dataDirectory, schema); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestSharedGlueMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestSharedGlueMetastore.java index 7b0d1a0f60f9..fee1e0c300ce 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestSharedGlueMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestSharedGlueMetastore.java @@ -29,14 +29,13 @@ import io.trino.plugin.hive.metastore.glue.DefaultGlueColumnStatisticsProviderFactory; import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore; import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig; +import io.trino.plugin.iceberg.BaseSharedMetastoreTest; import io.trino.plugin.iceberg.IcebergPlugin; import io.trino.plugin.tpch.TpchPlugin; -import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; import io.trino.tpch.TpchTable; import org.testng.annotations.AfterClass; -import org.testng.annotations.Test; import java.nio.file.Path; import java.util.Optional; @@ -46,25 +45,20 @@ import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static io.trino.testing.QueryAssertions.copyTpchTables; import static io.trino.testing.TestingSession.testSessionBuilder; -import static io.trino.testing.sql.TestTable.randomTableSuffix; import static java.lang.String.format; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.testng.Assert.assertEquals; /** * Tests metadata operations on a schema which has a mix of Hive and Iceberg tables. - * + *

* Requires AWS credentials, which can be provided any way supported by the DefaultProviderChain * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default */ public class TestSharedGlueMetastore - extends AbstractTestQueryFramework + extends BaseSharedMetastoreTest { private static final Logger LOG = Logger.get(TestSharedGlueMetastore.class); private static final String HIVE_CATALOG = "hive"; - private final String schema = "test_shared_glue_schema_" + randomTableSuffix(); private Path dataDirectory; private HiveMetastore glueMetastore; @@ -96,6 +90,13 @@ protected QueryRunner createQueryRunner() ImmutableMap.of( "iceberg.catalog.type", "glue", "hive.metastore.glue.default-warehouse-dir", dataDirectory.toString())); + queryRunner.createCatalog( + "iceberg_with_redirections", + "iceberg", + ImmutableMap.of( + "iceberg.catalog.type", "glue", + "hive.metastore.glue.default-warehouse-dir", dataDirectory.toString(), + "iceberg.hive-catalog-name", "hive")); HdfsConfig hdfsConfig = new HdfsConfig(); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment( @@ -138,99 +139,25 @@ public void cleanup() } } - @Test - public void testReadInformationSchema() - { - assertThat(query("SELECT table_schema FROM hive.information_schema.tables WHERE table_name = 'region'")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - assertThat(query("SELECT table_schema FROM iceberg.information_schema.tables WHERE table_name = 'nation'")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - assertThat(query("SELECT table_schema FROM hive_with_redirections.information_schema.tables WHERE table_name = 'region'")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - assertThat(query("SELECT table_schema FROM hive_with_redirections.information_schema.tables WHERE table_name = 'nation'")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - - assertQuery("SELECT table_name, column_name from hive.information_schema.columns WHERE table_schema = '" + schema + "'", - "VALUES ('region', 'regionkey'), ('region', 'name'), ('region', 'comment')"); - assertQuery("SELECT table_name, column_name from iceberg.information_schema.columns WHERE table_schema = '" + schema + "'", - "VALUES ('nation', 'nationkey'), ('nation', 'name'), ('nation', 'regionkey'), ('nation', 'comment')"); - assertQuery("SELECT table_name, column_name from hive_with_redirections.information_schema.columns WHERE table_schema = '" + schema + "'", - "VALUES" + - "('region', 'regionkey'), ('region', 'name'), ('region', 'comment'), " + - "('nation', 'nationkey'), ('nation', 'name'), ('nation', 'regionkey'), ('nation', 'comment')"); - } - - @Test - public void testShowTables() - { - assertQuery("SHOW TABLES FROM iceberg." + schema, "VALUES 'region', 'nation'"); - assertQuery("SHOW TABLES FROM hive." + schema, "VALUES 'region', 'nation'"); - assertQuery("SHOW TABLES FROM hive_with_redirections." + schema, "VALUES 'region', 'nation'"); - - assertThatThrownBy(() -> query("SHOW CREATE TABLE iceberg." + schema + ".region")) - .hasMessageContaining("Not an Iceberg table"); - assertThatThrownBy(() -> query("SHOW CREATE TABLE hive." + schema + ".nation")) - .hasMessageContaining("Cannot query Iceberg table"); - - assertThatThrownBy(() -> query("DESCRIBE iceberg." + schema + ".region")) - .hasMessageContaining("Not an Iceberg table"); - assertThatThrownBy(() -> query("DESCRIBE hive." + schema + ".nation")) - .hasMessageContaining("Cannot query Iceberg table"); - } - - @Test - public void testShowSchemas() + @Override + protected String getExpectedHiveCreateSchema(String catalogName) { - assertThat(query("SHOW SCHEMAS FROM hive")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - assertThat(query("SHOW SCHEMAS FROM iceberg")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - assertThat(query("SHOW SCHEMAS FROM hive_with_redirections")) - .skippingTypesCheck() - .containsAll("VALUES '" + schema + "'"); - String expectedHiveCreateSchema = "CREATE SCHEMA %s.%s\n" + "AUTHORIZATION ROLE public\n" + "WITH (\n" + " location = '%s'\n" + ")"; - String showCreateHiveSchema = (String) computeActual("SHOW CREATE SCHEMA hive." + schema).getOnlyValue(); - assertEquals( - showCreateHiveSchema, - format(expectedHiveCreateSchema, "hive", schema, dataDirectory)); - String showCreateIcebergSchema = (String) computeActual("SHOW CREATE SCHEMA iceberg." + schema).getOnlyValue(); - assertEquals( - showCreateIcebergSchema, - format("CREATE SCHEMA iceberg.%s\n" + - "WITH (\n" + - " location = '%s'\n" + - ")", - schema, - dataDirectory)); - String showCreateHiveWithRedirectionsSchema = (String) computeActual("SHOW CREATE SCHEMA hive_with_redirections." + schema).getOnlyValue(); - assertEquals( - showCreateHiveWithRedirectionsSchema, - format(expectedHiveCreateSchema, "hive_with_redirections", schema, dataDirectory)); + return format(expectedHiveCreateSchema, catalogName, schema, dataDirectory); } - @Test - public void testSelect() + @Override + protected String getExpectedIcebergCreateSchema(String catalogName) { - assertQuery("SELECT * FROM iceberg." + schema + ".nation", "SELECT * FROM nation"); - assertQuery("SELECT * FROM hive." + schema + ".region", "SELECT * FROM region"); - assertQuery("SELECT * FROM hive_with_redirections." + schema + ".nation", "SELECT * FROM nation"); - assertQuery("SELECT * FROM hive_with_redirections." + schema + ".region", "SELECT * FROM region"); - - assertThatThrownBy(() -> query("SELECT * FROM iceberg." + schema + ".region")) - .hasMessageContaining("Not an Iceberg table"); - assertThatThrownBy(() -> query("SELECT * FROM hive." + schema + ".nation")) - .hasMessageContaining("Cannot query Iceberg table"); + String expectedIcebergCreateSchema = "CREATE SCHEMA %s.%s\n" + + "WITH (\n" + + " location = '%s'\n" + + ")"; + return format(expectedIcebergCreateSchema, catalogName, schema, dataDirectory, schema); } } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/iceberg.properties index 6230d550a4ac..13657dff5572 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/iceberg.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/iceberg.properties @@ -1,3 +1,4 @@ connector.name=iceberg hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +iceberg.hive-catalog-name=hive diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergHiveTablesCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergHiveTablesCompatibility.java index a84ac70063e2..05c16d9a6e54 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergHiveTablesCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergHiveTablesCompatibility.java @@ -44,9 +44,12 @@ public void testIcebergSelectFromHiveTable() assertQueryFailure(() -> onTrino().executeQuery("SELECT * FROM iceberg.default." + tableName)) .hasMessageMatching("Query failed \\(#\\w+\\):\\Q Not an Iceberg table: default." + tableName); - assertQueryFailure(() -> onTrino().executeQuery("SELECT * FROM iceberg.default.\"" + tableName + "$files\"")) + assertQueryFailure(() -> onTrino().executeQuery("SELECT * FROM iceberg.default.\"" + tableName + "$data\"")) .hasMessageMatching("Query failed \\(#\\w+\\):\\Q Not an Iceberg table: default." + tableName); + assertQueryFailure(() -> onTrino().executeQuery("SELECT * FROM iceberg.default.\"" + tableName + "$files\"")) + .hasMessageMatching("Query failed \\(#\\w+\\):\\Q line 1:15: Table 'iceberg.default." + tableName + "$files' does not exist"); + onTrino().executeQuery("DROP TABLE hive.default." + tableName); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergRedirectionToHive.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergRedirectionToHive.java index 953c2467e93d..89b927502073 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergRedirectionToHive.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergRedirectionToHive.java @@ -59,9 +59,9 @@ public void testRedirect() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("TABLE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName), + onTrino().executeQuery("TABLE " + icebergTableName)); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -75,9 +75,9 @@ public void testRedirectWithNonDefaultSchema() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("TABLE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: nondefaultschema." + tableName); + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName), + onTrino().executeQuery("TABLE " + icebergTableName)); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -91,12 +91,15 @@ public void testRedirectToNonexistentCatalog() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("SET SESSION iceberg.hive_catalog_name = 'someweirdcatalog'")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Session property 'iceberg.hive_catalog_name' does not exist"); + // sanity check + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName), + onTrino().executeQuery("TABLE " + icebergTableName)); + + onTrino().executeQuery("SET SESSION iceberg.hive_catalog_name = 'someweirdcatalog'"); assertQueryFailure(() -> onTrino().executeQuery("TABLE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + .hasMessageMatching(".*Table 'iceberg.default.redirect_to_nonexistent_hive_.*' redirected to 'someweirdcatalog.default.redirect_to_nonexistent_hive_.*', but the target catalog 'someweirdcatalog' does not exist"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -112,20 +115,14 @@ public void testRedirectWithDefaultSchemaInSession() createHiveTable(hiveTableName, false); onTrino().executeQuery("USE iceberg.default"); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("TABLE " + tableName)) // unqualified - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); -// assertResultsEqual( -// onTrino().executeQuery("TABLE " + tableName), // unqualified -// onTrino().executeQuery("TABLE " + hiveTableName)); + assertResultsEqual( + onTrino().executeQuery("TABLE " + tableName), // unqualified + onTrino().executeQuery("TABLE " + hiveTableName)); onTrino().executeQuery("USE hive.default"); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("TABLE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); -// assertResultsEqual( -// onTrino().executeQuery("TABLE " + icebergTableName), -// onTrino().executeQuery("TABLE " + tableName)); // unqualified + assertResultsEqual( + onTrino().executeQuery("TABLE " + icebergTableName), + onTrino().executeQuery("TABLE " + tableName)); // unqualified onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -135,12 +132,12 @@ public void testRedirectPartitionsToUnpartitioned() { String tableName = "hive_unpartitioned_table_" + randomTableSuffix(); String hiveTableName = "hive.default." + tableName; + String icebergTableName = "iceberg.default." + tableName; createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive assertQueryFailure(() -> onTrino().executeQuery("TABLE iceberg.default.\"" + tableName + "$partitions\"")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Table '" + icebergTableName + "$partitions' redirected to '" + hiveTableName + "$partitions', but the target table '" + hiveTableName + "$partitions' does not exist"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -153,9 +150,13 @@ public void testRedirectPartitionsToPartitioned() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("TABLE iceberg.default.\"" + tableName + "$partitions\"")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertThat(onTrino().executeQuery("TABLE iceberg.default.\"" + tableName + "$partitions\"")) + .containsOnly( + row(0), + row(1), + row(2), + row(3), + row(4)); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -169,9 +170,12 @@ public void testInsert(String schema, boolean partitioned) createHiveTable(hiveTableName, partitioned, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO " + icebergTableName + " VALUES (6, false, -17), (7, true, 1)")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: " + schema + "." + tableName); + onTrino().executeQuery("INSERT INTO " + icebergTableName + " VALUES (42, 'some name', 'some comment', 12)"); + + assertThat(onTrino().executeQuery("TABLE " + hiveTableName)) + .containsOnly(row(42L, "some name", "some comment", 12L)); + assertThat(onTrino().executeQuery("TABLE " + icebergTableName)) + .containsOnly(row(42L, "some name", "some comment", 12L)); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -197,9 +201,11 @@ public void testDelete() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM " + icebergTableName + " WHERE regionkey = 1")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + onTrino().executeQuery("DELETE FROM " + icebergTableName + " WHERE regionkey = 1"); + + assertResultsEqual( + onTrino().executeQuery("TABLE " + icebergTableName), + onTrino().executeQuery("SELECT nationkey, name, comment, regionkey FROM tpch.tiny.nation WHERE regionkey != 1")); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -213,9 +219,8 @@ public void testUpdate() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive assertQueryFailure(() -> onTrino().executeQuery("UPDATE " + icebergTableName + " SET nationkey = nationkey + 100 WHERE regionkey = 1")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Hive update is only supported for ACID transactional tables"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -228,11 +233,9 @@ public void testDropTable() String icebergTableName = "iceberg.default." + tableName; createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("DROP TABLE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); - // TODO should fail - onTrino().executeQuery("TABLE " + hiveTableName); + onTrino().executeQuery("DROP TABLE " + icebergTableName); + assertQueryFailure(() -> onTrino().executeQuery("TABLE " + hiveTableName)) + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table '" + hiveTableName + "' does not exist"); } @Test(groups = {HIVE_ICEBERG_REDIRECTIONS, PROFILE_SPECIFIC_TESTS}) @@ -244,9 +247,9 @@ public void testDescribe() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("DESCRIBE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertResultsEqual( + onTrino().executeQuery("DESCRIBE " + icebergTableName), + onTrino().executeQuery("DESCRIBE " + hiveTableName)); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -260,9 +263,17 @@ public void testShowCreateTable() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("SHOW CREATE TABLE " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertThat(onTrino().executeQuery("SHOW CREATE TABLE " + icebergTableName)) + .containsOnly(row("CREATE TABLE " + hiveTableName + " (\n" + + " nationkey bigint,\n" + + " name varchar(25),\n" + + " comment varchar(152),\n" + + " regionkey bigint\n" + + ")\n" + + "WITH (\n" + + " format = 'ORC',\n" + + " partitioned_by = ARRAY['regionkey']\n" + // 'partitioned_by' comes from Hive + ")")); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -276,9 +287,13 @@ public void testShowStats() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("SHOW STATS FOR " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertThat(onTrino().executeQuery("SHOW STATS FOR " + icebergTableName)) + .containsOnly( + row("nationkey", null, 5d, 0d, null, "0", "24"), + row("name", 177d, 5d, 0d, null, null, null), + row("comment", 1857d, 5d, 0d, null, null, null), + row("regionkey", null, 5d, 0d, null, "0", "4"), + row(null, null, null, null, 25d, null, null)); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -286,17 +301,27 @@ public void testShowStats() @Test(groups = {HIVE_ICEBERG_REDIRECTIONS, PROFILE_SPECIFIC_TESTS}) public void testAlterTableRename() { - String tableName = "hive_rename_table_" + randomTableSuffix(); + String tableName = "iceberg_rename_table_" + randomTableSuffix(); String hiveTableName = "hive.default." + tableName; String icebergTableName = "iceberg.default." + tableName; createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE " + icebergTableName + " RENAME TO a_new_name")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE " + icebergTableName + " RENAME TO iceberg.default." + tableName + "_new")) + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table rename across catalogs is not supported"); - onTrino().executeQuery("DROP TABLE " + hiveTableName); + String newTableNameWithoutCatalogWithoutSchema = tableName + "_new_without_catalog_without_schema"; + onTrino().executeQuery("ALTER TABLE " + icebergTableName + " RENAME TO " + newTableNameWithoutCatalogWithoutSchema); + String newTableNameWithoutCatalogWithSchema = tableName + "_new_without_catalog_with_schema"; + onTrino().executeQuery("ALTER TABLE iceberg.default." + newTableNameWithoutCatalogWithoutSchema + " RENAME TO default." + newTableNameWithoutCatalogWithSchema); + String newTableNameWithCatalogWithSchema = tableName + "_new_with_catalog_with_schema"; + onTrino().executeQuery("ALTER TABLE iceberg.default." + newTableNameWithoutCatalogWithSchema + " RENAME TO hive.default." + newTableNameWithCatalogWithSchema); + + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName + "_new_with_catalog_with_schema"), + onTrino().executeQuery("TABLE " + icebergTableName + "_new_with_catalog_with_schema")); + + onTrino().executeQuery("DROP TABLE " + hiveTableName + "_new_with_catalog_with_schema"); } @Test(groups = {HIVE_ICEBERG_REDIRECTIONS, PROFILE_SPECIFIC_TESTS}) @@ -308,10 +333,14 @@ public void testAlterTableAddColumn() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE " + icebergTableName + " ADD COLUMN some_new_column double")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + onTrino().executeQuery("ALTER TABLE " + icebergTableName + " ADD COLUMN some_new_column double"); + + Assertions.assertThat(onTrino().executeQuery("DESCRIBE " + hiveTableName).column(1)) + .containsOnly("nationkey", "name", "regionkey", "comment", "some_new_column"); + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName), + onTrino().executeQuery("SELECT nationkey, name, comment, regionkey, NULL FROM tpch.tiny.nation")); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -324,9 +353,14 @@ public void testAlterTableRenameColumn() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE " + icebergTableName + " RENAME COLUMN nationkey TO nation_key")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + onTrino().executeQuery("ALTER TABLE " + icebergTableName + " RENAME COLUMN nationkey TO nation_key"); + + Assertions.assertThat(onTrino().executeQuery("DESCRIBE " + icebergTableName).column(1)) + .containsOnly("nation_key", "name", "regionkey", "comment"); + + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName), + onTrino().executeQuery("SELECT nationkey as nation_key, name, comment, regionkey FROM tpch.tiny.nation")); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -340,10 +374,17 @@ public void testAlterTableDropColumn() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE " + icebergTableName + " DROP COLUMN comment")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + onTrino().executeQuery("ALTER TABLE " + icebergTableName + " DROP COLUMN comment"); + + Assertions.assertThat(onTrino().executeQuery("DESCRIBE " + icebergTableName).column(1)) + .containsOnly("nationkey", "name", "regionkey"); + // After dropping the column from the Hive metastore, access ORC columns by name + // in order to avoid mismatches between column indexes + onTrino().executeQuery("SET SESSION hive.orc_use_column_names = true"); + assertResultsEqual( + onTrino().executeQuery("TABLE " + hiveTableName), + onTrino().executeQuery("SELECT nationkey, name, regionkey FROM tpch.tiny.nation")); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -359,12 +400,11 @@ public void testCommentTable() assertTableComment("hive", "default", tableName).isNull(); assertTableComment("iceberg", "default", tableName).isNull(); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("COMMENT ON TABLE " + icebergTableName + " IS 'This is my table, there are many like it but this one is mine'")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + String tableComment = "This is my table, there are many like it but this one is mine"; + onTrino().executeQuery(format("COMMENT ON TABLE " + icebergTableName + " IS '%s'", tableComment)); - assertTableComment("hive", "default", tableName).isNull(); - assertTableComment("iceberg", "default", tableName).isNull(); + assertTableComment("hive", "default", tableName).isEqualTo(tableComment); + assertTableComment("iceberg", "default", tableName).isEqualTo(tableComment); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -378,9 +418,8 @@ public void testShowGrants() createHiveTable(hiveTableName, true); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("SHOW GRANTS ON " + icebergTableName)) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertQueryFailure(() -> onTrino().executeQuery(format("SHOW GRANTS ON %s", icebergTableName))) + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table " + icebergTableName + " is redirected to " + hiveTableName + " and SHOW GRANTS is not supported with table redirections"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -398,21 +437,22 @@ public void testInformationSchemaColumns() createHiveTable(hiveTableName, false); // via redirection with table filter - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery( + assertThat(onTrino().executeQuery( format("SELECT * FROM iceberg.information_schema.columns WHERE table_schema = '%s' AND table_name='%s'", schemaName, tableName))) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: " + schemaName + "." + tableName); + .containsOnly( + row("iceberg", schemaName, tableName, "nationkey", 1, null, "YES", "bigint"), + row("iceberg", schemaName, tableName, "name", 2, null, "YES", "varchar(25)"), + row("iceberg", schemaName, tableName, "comment", 3, null, "YES", "varchar(152)"), + row("iceberg", schemaName, tableName, "regionkey", 4, null, "YES", "bigint")); // test via redirection with just schema filter assertThat(onTrino().executeQuery( format("SELECT * FROM iceberg.information_schema.columns WHERE table_schema = '%s'", schemaName))) .containsOnly( - // TODO report table's columns via redirect to Hive -// row("iceberg", schemaName, tableName, "nationkey", 1, null, "YES", "bigint"), -// row("iceberg", schemaName, tableName, "name", 2, null, "YES", "varchar(25)"), -// row("iceberg", schemaName, tableName, "comment", 3, null, "YES", "varchar(152)"), -// row("iceberg", schemaName, tableName, "regionkey", 4, null, "YES", "bigint") - /**/); + row("iceberg", schemaName, tableName, "nationkey", 1, null, "YES", "bigint"), + row("iceberg", schemaName, tableName, "name", 2, null, "YES", "varchar(25)"), + row("iceberg", schemaName, tableName, "comment", 3, null, "YES", "varchar(152)"), + row("iceberg", schemaName, tableName, "regionkey", 4, null, "YES", "bigint")); // sanity check that getting columns info without redirection produces matching result assertThat(onTrino().executeQuery( @@ -443,23 +483,19 @@ public void testSystemJdbcColumns() assertThat(onTrino().executeQuery( format("SELECT table_cat, table_schem, table_name, column_name FROM system.jdbc.columns WHERE table_cat = 'iceberg' AND table_schem = '%s' AND table_name = '%s'", schemaName, tableName))) .containsOnly( - // TODO report table's columns via redirect to Hive -// row("iceberg", schemaName, tableName, "nationkey"), -// row("iceberg", schemaName, tableName, "name"), -// row("iceberg", schemaName, tableName, "comment"), -// row("iceberg", schemaName, tableName, "regionkey") - /**/); + row("iceberg", schemaName, tableName, "nationkey"), + row("iceberg", schemaName, tableName, "name"), + row("iceberg", schemaName, tableName, "comment"), + row("iceberg", schemaName, tableName, "regionkey")); - // test via redirection with just schema filter + // test via redirection with just schema filter - consistent with the functionality of the command `SHOW TABLES` command on the Iceberg connector assertThat(onTrino().executeQuery( format("SELECT table_cat, table_schem, table_name, column_name FROM system.jdbc.columns WHERE table_cat = 'iceberg' AND table_schem = '%s'", schemaName))) .containsOnly( - // TODO report table's columns via redirect to Hive -// row("iceberg", schemaName, tableName, "nationkey"), -// row("iceberg", schemaName, tableName, "name"), -// row("iceberg", schemaName, tableName, "comment"), -// row("iceberg", schemaName, tableName, "regionkey") - /**/); + row("iceberg", schemaName, tableName, "nationkey"), + row("iceberg", schemaName, tableName, "name"), + row("iceberg", schemaName, tableName, "comment"), + row("iceberg", schemaName, tableName, "regionkey")); // sanity check that getting columns info without redirection produces matching result assertThat(onTrino().executeQuery( @@ -483,9 +519,8 @@ public void testGrant() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive assertQueryFailure(() -> onTrino().executeQuery("GRANT SELECT ON " + icebergTableName + " TO ROLE PUBLIC")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table " + icebergTableName + " is redirected to " + hiveTableName + " and GRANT is not supported with table redirections"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -499,9 +534,8 @@ public void testRevoke() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive assertQueryFailure(() -> onTrino().executeQuery("REVOKE SELECT ON " + icebergTableName + " FROM ROLE PUBLIC")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table " + icebergTableName + " is redirected to " + hiveTableName + " and REVOKE is not supported with table redirections"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -515,9 +549,8 @@ public void testSetTableAuthorization() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE " + icebergTableName + " SET AUTHORIZATION ROLE PUBLIC")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table " + icebergTableName + " is redirected to " + hiveTableName + " and SET TABLE AUTHORIZATION is not supported with table redirections"); onTrino().executeQuery("DROP TABLE " + hiveTableName); } @@ -531,9 +564,8 @@ public void testDeny() createHiveTable(hiveTableName, false); - // TODO: support redirects from Iceberg to Hive - assertQueryFailure(() -> onTrino().executeQuery("GRANT SELECT ON " + icebergTableName + " TO ROLE PUBLIC")) - .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): Not an Iceberg table: default." + tableName); + assertQueryFailure(() -> onTrino().executeQuery("DENY DELETE ON " + icebergTableName + " TO ROLE PUBLIC")) + .hasMessageMatching("\\QQuery failed (#\\E\\S+\\Q): line 1:1: Table " + icebergTableName + " is redirected to " + hiveTableName + " and DENY is not supported with table redirections"); onTrino().executeQuery("DROP TABLE " + hiveTableName); }