diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java index 976f4cced93d..099a8243d610 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java @@ -134,6 +134,9 @@ public class HiveConfig private boolean projectionPushdownEnabled = true; + private boolean redirectToIcebergEnabled; + private String redirectToIcebergCatalog = "iceberg"; + public int getMaxInitialSplits() { return maxInitialSplits; @@ -977,4 +980,31 @@ public HiveConfig setProjectionPushdownEnabled(boolean projectionPushdownEnabled this.projectionPushdownEnabled = projectionPushdownEnabled; return this; } + + public boolean isRedirectToIcebergEnabled() + { + return redirectToIcebergEnabled; + } + + @Config("hive.redirect-to-iceberg-enabled") + @ConfigDescription("Redirect to a catalog configured with Iceberg Connector") + public HiveConfig setRedirectToIcebergEnabled(boolean redirectToIcebergEnabled) + { + this.redirectToIcebergEnabled = redirectToIcebergEnabled; + return this; + } + + @NotNull + public String getRedirectToIcebergCatalog() + { + return redirectToIcebergCatalog; + } + + @Config("hive.redirect-to-iceberg-catalog") + @ConfigDescription("The Iceberg catalog to redirect to") + public HiveConfig setRedirectToIcebergCatalog(String redirectToIcebergCatalog) + { + this.redirectToIcebergCatalog = redirectToIcebergCatalog; + return this; + } } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java index df25b3c2be3f..12a8e20e717b 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java @@ -120,6 +120,8 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -159,11 +161,13 @@ import static io.prestosql.plugin.hive.HivePartitionManager.extractPartitionValues; import static io.prestosql.plugin.hive.HiveSessionProperties.getCompressionCodec; import static io.prestosql.plugin.hive.HiveSessionProperties.getHiveStorageFormat; +import static io.prestosql.plugin.hive.HiveSessionProperties.getRedirectToIcebergCatalog; import static io.prestosql.plugin.hive.HiveSessionProperties.isBucketExecutionEnabled; import static io.prestosql.plugin.hive.HiveSessionProperties.isCollectColumnStatisticsOnWrite; import static io.prestosql.plugin.hive.HiveSessionProperties.isCreateEmptyBucketFiles; import static io.prestosql.plugin.hive.HiveSessionProperties.isOptimizedMismatchedBucketCount; import static io.prestosql.plugin.hive.HiveSessionProperties.isProjectionPushdownEnabled; +import static io.prestosql.plugin.hive.HiveSessionProperties.isRedirectToIcebergEnabled; import static io.prestosql.plugin.hive.HiveSessionProperties.isRespectTableFormat; import static io.prestosql.plugin.hive.HiveSessionProperties.isSortedWritingEnabled; import static io.prestosql.plugin.hive.HiveSessionProperties.isStatisticsEnabled; @@ -279,6 +283,8 @@ public class HiveMetadata public static final String AVRO_SCHEMA_URL_KEY = "avro.schema.url"; public static final String SPARK_TABLE_PROVIDER_KEY = "spark.sql.sources.provider"; public static final String DELTA_LAKE_PROVIDER = "delta"; + public static final String TABLE_TYPE_PROP = "table_type"; + public static final String ICEBERG_TABLE_TYPE_VALUE = "iceberg"; private static final String CSV_SEPARATOR_KEY = OpenCSVSerde.SEPARATORCHAR; private static final String CSV_QUOTE_KEY = OpenCSVSerde.QUOTECHAR; @@ -298,6 +304,12 @@ public class HiveMetadata private final HiveStatisticsProvider hiveStatisticsProvider; private final AccessControlMetadata accessControlMetadata; + // Copied from IcebergTableHandle + private static final Pattern ICEBERG_TABLE_NAME_PATTERN = Pattern.compile("" + + "(?[^$@]+)" + + "(?:@(?[0-9]+))?" + + "(?:\\$(?[^@]+)(?:@(?[0-9]+))?)?"); + public HiveMetadata( CatalogName catalogName, SemiTransactionalHiveMetastore metastore, @@ -2654,6 +2666,37 @@ public void cleanupQuery(ConnectorSession session) metastore.cleanupQuery(session); } + @Override + public Optional redirectCatalog(ConnectorSession session, SchemaTableName tableName) + { + if (!isRedirectToIcebergEnabled(session)) { + return Optional.empty(); + } + if (!filterSchema(tableName.getSchemaName())) { + return Optional.empty(); + } + String table = tableName.getTableName(); + Matcher icebergNameMatch = ICEBERG_TABLE_NAME_PATTERN.matcher(table); + if (icebergNameMatch.matches()) { + table = icebergNameMatch.group("table"); + } + Optional
hiveTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), table); + if (hiveTable.isEmpty()) { + return Optional.empty(); + } + + if (isIcebergTable(hiveTable.get())) { + return Optional.of(getRedirectToIcebergCatalog(session)); + } + + return Optional.empty(); + } + + private static boolean isIcebergTable(Table table) + { + return ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getParameters().get(TABLE_TYPE_PROP)); + } + public static Optional getSourceTableNameFromSystemTable(SchemaTableName tableName) { return Stream.of(SystemTableHandler.values()) diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java index c7cfeed4adf2..9d893e4a4cae 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java @@ -91,6 +91,8 @@ public final class HiveSessionProperties private static final String QUERY_PARTITION_FILTER_REQUIRED = "query_partition_filter_required"; private static final String PROJECTION_PUSHDOWN_ENABLED = "projection_pushdown_enabled"; private static final String PARQUET_OPTIMIZED_WRITER_ENABLED = "parquet_optimized_writer_enabled"; + private static final String REDIRECT_TO_ICEBERG_ENABLED = "redirect_to_iceberg_enabled"; + private static final String REDIRECT_TO_ICEBERG_CATALOG = "redirect_to_iceberg_catalog"; private final List> sessionProperties; @@ -369,6 +371,16 @@ public HiveSessionProperties( PARQUET_OPTIMIZED_WRITER_ENABLED, "Experimental: Enable optimized writer", parquetWriterConfig.isParquetOptimizedWriterEnabled(), + false), + booleanProperty( + REDIRECT_TO_ICEBERG_ENABLED, + "Enable redirecting to a catalog configured with Iceberg Connector", + hiveConfig.isRedirectToIcebergEnabled(), + false), + stringProperty( + REDIRECT_TO_ICEBERG_CATALOG, + "The target Iceberg catalog for redirection", + hiveConfig.getRedirectToIcebergCatalog(), false)); } @@ -635,4 +647,14 @@ public static boolean isParquetOptimizedWriterEnabled(ConnectorSession session) { return session.getProperty(PARQUET_OPTIMIZED_WRITER_ENABLED, Boolean.class); } + + public static boolean isRedirectToIcebergEnabled(ConnectorSession session) + { + return session.getProperty(REDIRECT_TO_ICEBERG_ENABLED, Boolean.class); + } + + public static String getRedirectToIcebergCatalog(ConnectorSession session) + { + return session.getProperty(REDIRECT_TO_ICEBERG_CATALOG, String.class); + } } diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java index 5cfb761fec92..c7413d8d17b9 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java @@ -96,7 +96,9 @@ public void testDefaults() .setAllowRegisterPartition(false) .setQueryPartitionFilterRequired(false) .setPartitionUseColumnNames(false) - .setProjectionPushdownEnabled(true)); + .setProjectionPushdownEnabled(true) + .setRedirectToIcebergEnabled(false) + .setRedirectToIcebergCatalog("iceberg")); } @Test @@ -166,6 +168,8 @@ public void testExplicitPropertyMappings() .put("hive.query-partition-filter-required", "true") .put("hive.partition-use-column-names", "true") .put("hive.projection-pushdown-enabled", "false") + .put("hive.redirect-to-iceberg-enabled", "true") + .put("hive.redirect-to-iceberg-catalog", "myiceberg") .build(); HiveConfig expected = new HiveConfig() @@ -231,7 +235,9 @@ public void testExplicitPropertyMappings() .setAllowRegisterPartition(true) .setQueryPartitionFilterRequired(true) .setPartitionUseColumnNames(true) - .setProjectionPushdownEnabled(false); + .setProjectionPushdownEnabled(false) + .setRedirectToIcebergEnabled(true) + .setRedirectToIcebergCatalog("myiceberg"); assertFullMapping(properties, expected); } diff --git a/presto-main/src/main/java/io/prestosql/execution/AddColumnTask.java b/presto-main/src/main/java/io/prestosql/execution/AddColumnTask.java index 1a043763bc48..25534e086a52 100644 --- a/presto-main/src/main/java/io/prestosql/execution/AddColumnTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/AddColumnTask.java @@ -36,6 +36,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.COLUMN_ALREADY_EXISTS; import static io.prestosql.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; @@ -64,6 +65,7 @@ public ListenableFuture execute(AddColumn statement, TransactionManager trans { Session session = stateMachine.getSession(); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isEmpty()) { if (!statement.isTableExists()) { @@ -72,8 +74,9 @@ public ListenableFuture execute(AddColumn statement, TransactionManager trans return immediateFuture(null); } - CatalogName catalogName = metadata.getCatalogHandle(session, tableName.getCatalogName()) - .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName())); + String catalog = tableName.getCatalogName(); + CatalogName catalogName = metadata.getCatalogHandle(session, catalog) + .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + catalog)); accessControl.checkCanAddColumns(session.toSecurityContext(), tableName); diff --git a/presto-main/src/main/java/io/prestosql/execution/CommentTask.java b/presto-main/src/main/java/io/prestosql/execution/CommentTask.java index 09e232853d25..3cf7b766c12c 100644 --- a/presto-main/src/main/java/io/prestosql/execution/CommentTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/CommentTask.java @@ -31,6 +31,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.MISSING_TABLE; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; @@ -53,6 +54,7 @@ public ListenableFuture execute(Comment statement, TransactionManager transac if (statement.getType() == Comment.Type.TABLE) { QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isEmpty()) { throw semanticException(TABLE_NOT_FOUND, statement, "Table does not exist: %s", tableName); @@ -69,6 +71,7 @@ else if (statement.getType() == Comment.Type.COLUMN) { } QualifiedObjectName tableName = createQualifiedObjectName(session, statement, prefix.get()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isEmpty()) { throw semanticException(TABLE_NOT_FOUND, statement, "Table does not exist: " + tableName); diff --git a/presto-main/src/main/java/io/prestosql/execution/CreateTableTask.java b/presto-main/src/main/java/io/prestosql/execution/CreateTableTask.java index 192df52a2fbc..e13546b1bf02 100644 --- a/presto-main/src/main/java/io/prestosql/execution/CreateTableTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/CreateTableTask.java @@ -51,6 +51,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.ALREADY_EXISTS; import static io.prestosql.spi.StandardErrorCode.CATALOG_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; @@ -98,6 +99,7 @@ ListenableFuture internalExecute(CreateTable statement, Metadata metadata, Ac Map, Expression> parameterLookup = parameterExtractor(statement, parameters); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isPresent()) { if (!statement.isNotExists()) { @@ -106,8 +108,9 @@ ListenableFuture internalExecute(CreateTable statement, Metadata metadata, Ac return immediateFuture(null); } - CatalogName catalogName = metadata.getCatalogHandle(session, tableName.getCatalogName()) - .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName())); + String catalog = tableName.getCatalogName(); + CatalogName catalogName = metadata.getCatalogHandle(session, catalog) + .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + catalog)); LinkedHashMap columns = new LinkedHashMap<>(); Map inheritedProperties = ImmutableMap.of(); @@ -154,14 +157,16 @@ ListenableFuture internalExecute(CreateTable statement, Metadata metadata, Ac else if (element instanceof LikeClause) { LikeClause likeClause = (LikeClause) element; QualifiedObjectName likeTableName = createQualifiedObjectName(session, statement, likeClause.getTableName()); + likeTableName = redirectToNewCatalogIfNecessary(session, likeTableName, metadata); if (metadata.getCatalogHandle(session, likeTableName.getCatalogName()).isEmpty()) { throw semanticException(CATALOG_NOT_FOUND, statement, "LIKE table catalog '%s' does not exist", likeTableName.getCatalogName()); } if (!tableName.getCatalogName().equals(likeTableName.getCatalogName())) { throw semanticException(NOT_SUPPORTED, statement, "LIKE table across catalogs is not supported"); } + QualifiedObjectName finalLikeTableName = likeTableName; TableHandle likeTable = metadata.getTableHandle(session, likeTableName) - .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, statement, "LIKE table '%s' does not exist", likeTableName)); + .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, statement, "LIKE table '%s' does not exist", finalLikeTableName)); TableMetadata likeTableMetadata = metadata.getTableMetadata(session, likeTable); diff --git a/presto-main/src/main/java/io/prestosql/execution/CreateViewTask.java b/presto-main/src/main/java/io/prestosql/execution/CreateViewTask.java index 3e701f8e48f3..fada267697e6 100644 --- a/presto-main/src/main/java/io/prestosql/execution/CreateViewTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/CreateViewTask.java @@ -37,6 +37,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.connector.ConnectorViewDefinition.ViewColumn; import static io.prestosql.sql.ParameterUtils.parameterExtractor; import static io.prestosql.sql.SqlFormatterUtil.getFormattedSql; @@ -72,7 +73,7 @@ public ListenableFuture execute(CreateView statement, TransactionManager tran { Session session = stateMachine.getSession(); QualifiedObjectName name = createQualifiedObjectName(session, statement, statement.getName()); - + name = redirectToNewCatalogIfNecessary(session, name, metadata); accessControl.checkCanCreateView(session.toSecurityContext(), name); String sql = getFormattedSql(statement.getQuery(), sqlParser); diff --git a/presto-main/src/main/java/io/prestosql/execution/DropColumnTask.java b/presto-main/src/main/java/io/prestosql/execution/DropColumnTask.java index 0e1468b77c67..75ef89a0b2b4 100644 --- a/presto-main/src/main/java/io/prestosql/execution/DropColumnTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/DropColumnTask.java @@ -29,6 +29,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; @@ -49,6 +50,7 @@ public ListenableFuture execute(DropColumn statement, TransactionManager tran { Session session = stateMachine.getSession(); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTable()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandleOptional = metadata.getTableHandle(session, tableName); if (tableHandleOptional.isEmpty()) { diff --git a/presto-main/src/main/java/io/prestosql/execution/DropTableTask.java b/presto-main/src/main/java/io/prestosql/execution/DropTableTask.java index 39bb82f360f7..cc6e3913be87 100644 --- a/presto-main/src/main/java/io/prestosql/execution/DropTableTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/DropTableTask.java @@ -28,6 +28,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; import static io.prestosql.sql.analyzer.SemanticExceptions.semanticException; @@ -45,6 +46,7 @@ public ListenableFuture execute(DropTable statement, TransactionManager trans { Session session = stateMachine.getSession(); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isEmpty()) { diff --git a/presto-main/src/main/java/io/prestosql/execution/DropViewTask.java b/presto-main/src/main/java/io/prestosql/execution/DropViewTask.java index 7e1f198c6c53..d423dde5c9e9 100644 --- a/presto-main/src/main/java/io/prestosql/execution/DropViewTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/DropViewTask.java @@ -28,6 +28,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; import static io.prestosql.sql.analyzer.SemanticExceptions.semanticException; @@ -45,6 +46,7 @@ public ListenableFuture execute(DropView statement, TransactionManager transa { Session session = stateMachine.getSession(); QualifiedObjectName name = createQualifiedObjectName(session, statement, statement.getName()); + name = redirectToNewCatalogIfNecessary(session, name, metadata); Optional view = metadata.getView(session, name); if (view.isEmpty()) { diff --git a/presto-main/src/main/java/io/prestosql/execution/GrantTask.java b/presto-main/src/main/java/io/prestosql/execution/GrantTask.java index 0bbc2bfe3157..5eab1baad6ba 100644 --- a/presto-main/src/main/java/io/prestosql/execution/GrantTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/GrantTask.java @@ -33,6 +33,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createPrincipal; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.INVALID_PRIVILEGE; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; import static io.prestosql.sql.analyzer.SemanticExceptions.semanticException; @@ -51,6 +52,7 @@ public ListenableFuture execute(Grant statement, TransactionManager transacti { Session session = stateMachine.getSession(); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isEmpty()) { throw semanticException(TABLE_NOT_FOUND, statement, "Table '%s' does not exist", tableName); diff --git a/presto-main/src/main/java/io/prestosql/execution/RenameColumnTask.java b/presto-main/src/main/java/io/prestosql/execution/RenameColumnTask.java index f48b3ce18b46..41777efaf8c1 100644 --- a/presto-main/src/main/java/io/prestosql/execution/RenameColumnTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/RenameColumnTask.java @@ -30,6 +30,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.COLUMN_ALREADY_EXISTS; import static io.prestosql.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; @@ -51,6 +52,7 @@ public ListenableFuture execute(RenameColumn statement, TransactionManager tr { Session session = stateMachine.getSession(); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTable()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandleOptional = metadata.getTableHandle(session, tableName); if (tableHandleOptional.isEmpty()) { if (!statement.isTableExists()) { diff --git a/presto-main/src/main/java/io/prestosql/execution/RenameTableTask.java b/presto-main/src/main/java/io/prestosql/execution/RenameTableTask.java index bf23480a57af..57acb8e232db 100644 --- a/presto-main/src/main/java/io/prestosql/execution/RenameTableTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/RenameTableTask.java @@ -28,7 +28,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; -import static io.prestosql.spi.StandardErrorCode.CATALOG_NOT_FOUND; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; import static io.prestosql.spi.StandardErrorCode.TABLE_ALREADY_EXISTS; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; @@ -47,26 +47,27 @@ public String getName() public ListenableFuture execute(RenameTable statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) { Session session = stateMachine.getSession(); - QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getSource()); - Optional tableHandle = metadata.getTableHandle(session, tableName); + QualifiedObjectName originalTableName = createQualifiedObjectName(session, statement, statement.getSource()); + QualifiedObjectName redirectedTableName = redirectToNewCatalogIfNecessary(session, originalTableName, metadata); + Optional tableHandle = metadata.getTableHandle(session, redirectedTableName); if (tableHandle.isEmpty()) { if (!statement.isExists()) { - throw semanticException(TABLE_NOT_FOUND, statement, "Table '%s' does not exist", tableName); + throw semanticException(TABLE_NOT_FOUND, statement, "Table '%s' does not exist", redirectedTableName); } return immediateFuture(null); } QualifiedObjectName target = createQualifiedObjectName(session, statement, statement.getTarget()); - if (metadata.getCatalogHandle(session, target.getCatalogName()).isEmpty()) { - throw semanticException(CATALOG_NOT_FOUND, statement, "Target catalog '%s' does not exist", target.getCatalogName()); + if (!redirectedTableName.getCatalogName().equals(target.getCatalogName())) { + if (!originalTableName.getCatalogName().equals(target.getCatalogName())) { + throw semanticException(NOT_SUPPORTED, statement, "Table rename across catalogs is not supported"); + } + target = new QualifiedObjectName(redirectedTableName.getCatalogName(), target.getSchemaName(), target.getObjectName()); } if (metadata.getTableHandle(session, target).isPresent()) { throw semanticException(TABLE_ALREADY_EXISTS, statement, "Target table '%s' already exists", target); } - if (!tableName.getCatalogName().equals(target.getCatalogName())) { - throw semanticException(NOT_SUPPORTED, statement, "Table rename across catalogs is not supported"); - } - accessControl.checkCanRenameTable(session.toSecurityContext(), tableName, target); + accessControl.checkCanRenameTable(session.toSecurityContext(), redirectedTableName, target); metadata.renameTable(session, tableHandle.get(), target); diff --git a/presto-main/src/main/java/io/prestosql/execution/RenameViewTask.java b/presto-main/src/main/java/io/prestosql/execution/RenameViewTask.java index 2294951c4a89..7c8e6b0135c0 100644 --- a/presto-main/src/main/java/io/prestosql/execution/RenameViewTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/RenameViewTask.java @@ -28,7 +28,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; -import static io.prestosql.spi.StandardErrorCode.CATALOG_NOT_FOUND; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; import static io.prestosql.spi.StandardErrorCode.TABLE_ALREADY_EXISTS; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; @@ -47,26 +47,27 @@ public String getName() public ListenableFuture execute(RenameView statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) { Session session = stateMachine.getSession(); - QualifiedObjectName viewName = createQualifiedObjectName(session, statement, statement.getSource()); - Optional viewDefinition = metadata.getView(session, viewName); + QualifiedObjectName originalViewName = createQualifiedObjectName(session, statement, statement.getSource()); + QualifiedObjectName redirectedViewName = redirectToNewCatalogIfNecessary(session, originalViewName, metadata); + Optional viewDefinition = metadata.getView(session, originalViewName); if (viewDefinition.isEmpty()) { - throw semanticException(TABLE_NOT_FOUND, statement, "View '%s' does not exist", viewName); + throw semanticException(TABLE_NOT_FOUND, statement, "View '%s' does not exist", originalViewName); } QualifiedObjectName target = createQualifiedObjectName(session, statement, statement.getTarget()); - if (metadata.getCatalogHandle(session, target.getCatalogName()).isEmpty()) { - throw semanticException(CATALOG_NOT_FOUND, statement, "Target catalog '%s' does not exist", target.getCatalogName()); + if (!redirectedViewName.getCatalogName().equals(target.getCatalogName())) { + if (!originalViewName.getCatalogName().equals(target.getCatalogName())) { + throw semanticException(NOT_SUPPORTED, statement, "View rename across catalogs is not supported"); + } + target = new QualifiedObjectName(redirectedViewName.getCatalogName(), target.getSchemaName(), target.getObjectName()); } if (metadata.getView(session, target).isPresent()) { throw semanticException(TABLE_ALREADY_EXISTS, statement, "Target view '%s' already exists", target); } - if (!viewName.getCatalogName().equals(target.getCatalogName())) { - throw semanticException(NOT_SUPPORTED, statement, "View rename across catalogs is not supported"); - } - accessControl.checkCanRenameView(session.toSecurityContext(), viewName, target); + accessControl.checkCanRenameView(session.toSecurityContext(), redirectedViewName, target); - metadata.renameView(session, viewName, target); + metadata.renameView(session, redirectedViewName, target); return immediateFuture(null); } diff --git a/presto-main/src/main/java/io/prestosql/execution/RevokeTask.java b/presto-main/src/main/java/io/prestosql/execution/RevokeTask.java index 36fe33ab746c..6707bba0338b 100644 --- a/presto-main/src/main/java/io/prestosql/execution/RevokeTask.java +++ b/presto-main/src/main/java/io/prestosql/execution/RevokeTask.java @@ -33,6 +33,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.prestosql.metadata.MetadataUtil.createPrincipal; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.INVALID_PRIVILEGE; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; import static io.prestosql.sql.analyzer.SemanticExceptions.semanticException; @@ -51,6 +52,7 @@ public ListenableFuture execute(Revoke statement, TransactionManager transact { Session session = stateMachine.getSession(); QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); Optional tableHandle = metadata.getTableHandle(session, tableName); if (tableHandle.isEmpty()) { throw semanticException(TABLE_NOT_FOUND, statement, "Table '%s' does not exist", tableName); diff --git a/presto-main/src/main/java/io/prestosql/metadata/Metadata.java b/presto-main/src/main/java/io/prestosql/metadata/Metadata.java index b87f39c88e0f..efd151c94126 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/Metadata.java +++ b/presto-main/src/main/java/io/prestosql/metadata/Metadata.java @@ -533,4 +533,9 @@ default ResolvedFunction getCoercion(Type fromType, Type toType) ColumnPropertyManager getColumnPropertyManager(); AnalyzePropertyManager getAnalyzePropertyManager(); + + /** + * Redirects to another catalog which may or may not use the same connector + */ + Optional redirectCatalog(Session session, QualifiedObjectName tableName); } diff --git a/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java b/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java index cc77a6a9493e..aae8b83157c7 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java +++ b/presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java @@ -1796,6 +1796,28 @@ public AnalyzePropertyManager getAnalyzePropertyManager() return analyzePropertyManager; } + @Override + public Optional redirectCatalog(Session session, QualifiedObjectName tableName) + { + requireNonNull(tableName, "tableName is null"); + + if (tableName.getCatalogName().isEmpty() || tableName.getSchemaName().isEmpty() || tableName.getObjectName().isEmpty()) { + return Optional.empty(); + } + + Optional catalog = getOptionalCatalogMetadata(session, tableName.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + CatalogName catalogName = catalogMetadata.getConnectorId(session, tableName); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(catalogName); + + ConnectorSession connectorSession = session.toConnectorSession(catalogName); + return metadata.redirectCatalog(connectorSession, tableName.asSchemaTableName()); + } + + return Optional.empty(); + } + // // Helpers // diff --git a/presto-main/src/main/java/io/prestosql/metadata/MetadataUtil.java b/presto-main/src/main/java/io/prestosql/metadata/MetadataUtil.java index 53e7b00b5419..f6646e4a0b22 100644 --- a/presto-main/src/main/java/io/prestosql/metadata/MetadataUtil.java +++ b/presto-main/src/main/java/io/prestosql/metadata/MetadataUtil.java @@ -31,10 +31,13 @@ import io.prestosql.sql.tree.PrincipalSpecification; import io.prestosql.sql.tree.QualifiedName; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static io.prestosql.spi.StandardErrorCode.MISSING_CATALOG_NAME; import static io.prestosql.spi.StandardErrorCode.MISSING_SCHEMA_NAME; import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; @@ -156,6 +159,24 @@ public static QualifiedObjectName createQualifiedObjectName(Session session, Nod return new QualifiedObjectName(catalogName, schemaName, objectName); } + public static QualifiedObjectName redirectToNewCatalogIfNecessary(Session session, QualifiedObjectName tableName, Metadata metadata) + { + requireNonNull(session, "session is null"); + requireNonNull(tableName, "tableName is null"); + requireNonNull(metadata, "metadata is null"); + Set visitedCatalogs = new HashSet<>(); + while (true) { + Optional redirectedCatalog = metadata.redirectCatalog(session, tableName); + if (redirectedCatalog.isEmpty()) { + return tableName; + } + if (!visitedCatalogs.add(redirectedCatalog.get())) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Catalog redirection forms a loop"); + } + tableName = new QualifiedObjectName(redirectedCatalog.get(), tableName.getSchemaName(), tableName.getObjectName()); + } + } + public static PrestoPrincipal createPrincipal(Session session, GrantorSpecification specification) { GrantorSpecification.Type type = specification.getType(); diff --git a/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java index cccecd6fa6a8..386b9fd6659a 100644 --- a/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java @@ -180,6 +180,7 @@ import static io.prestosql.metadata.FunctionKind.AGGREGATE; import static io.prestosql.metadata.FunctionKind.WINDOW; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.CATALOG_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; @@ -358,6 +359,7 @@ protected Scope visitUse(Use node, Optional scope) protected Scope visitInsert(Insert insert, Optional scope) { QualifiedObjectName targetTable = createQualifiedObjectName(session, insert, insert.getTarget()); + targetTable = redirectToNewCatalogIfNecessary(session, targetTable, metadata); if (metadata.getView(session, targetTable).isPresent()) { throw semanticException(NOT_SUPPORTED, insert, "Inserting into views is not supported"); } @@ -503,12 +505,14 @@ protected Scope visitDelete(Delete node, Optional scope) { Table table = node.getTable(); QualifiedObjectName tableName = createQualifiedObjectName(session, table, table.getName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); if (metadata.getView(session, tableName).isPresent()) { throw semanticException(NOT_SUPPORTED, node, "Deleting from views is not supported"); } + QualifiedObjectName finalTableName = tableName; TableHandle handle = metadata.getTableHandle(session, tableName) - .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, table, "Table '%s' does not exist", tableName)); + .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, table, "Table '%s' does not exist", finalTableName)); accessControl.checkCanDeleteFromTable(session.toSecurityContext(), tableName); @@ -546,6 +550,7 @@ protected Scope visitDelete(Delete node, Optional scope) protected Scope visitAnalyze(Analyze node, Optional scope) { QualifiedObjectName tableName = createQualifiedObjectName(session, node, node.getTableName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); analysis.setUpdateType("ANALYZE", tableName); // verify the target table exists and it's not a view @@ -554,8 +559,9 @@ protected Scope visitAnalyze(Analyze node, Optional scope) } validateProperties(node.getProperties(), scope); + QualifiedObjectName finalTableName = tableName; CatalogName catalogName = metadata.getCatalogHandle(session, tableName.getCatalogName()) - .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + tableName.getCatalogName())); + .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + finalTableName.getCatalogName())); Map analyzeProperties = metadata.getAnalyzePropertyManager().getProperties( catalogName, @@ -566,7 +572,7 @@ protected Scope visitAnalyze(Analyze node, Optional scope) accessControl, analysis.getParameters()); TableHandle tableHandle = metadata.getTableHandleForStatisticsCollection(session, tableName, analyzeProperties) - .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, node, "Table '%s' does not exist", tableName)); + .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, node, "Table '%s' does not exist", finalTableName)); // user must have read and insert permission in order to analyze stats of a table analysis.addTableColumnReferences( @@ -591,6 +597,7 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional targetTableHandle = metadata.getTableHandle(session, targetTable); @@ -637,8 +644,9 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional new PrestoException(NOT_FOUND, "Catalog does not exist: " + targetTable.getCatalogName())); + .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + finalTargetTable.getCatalogName())); Map properties = metadata.getTablePropertyManager().getProperties( catalogName, @@ -685,6 +693,7 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional scope) { QualifiedObjectName viewName = createQualifiedObjectName(session, node, node.getName()); + viewName = redirectToNewCatalogIfNecessary(session, viewName, metadata); analysis.setUpdateType("CREATE VIEW", viewName); // analyze the query that creates the view @@ -1044,6 +1053,7 @@ protected Scope visitTable(Table table, Optional scope) } QualifiedObjectName name = createQualifiedObjectName(session, table, table.getName()); + name = redirectToNewCatalogIfNecessary(session, name, metadata); analysis.addEmptyColumnReferencesForTable(accessControl, session.getIdentity(), name); // is this a reference to a view? @@ -1181,6 +1191,7 @@ private Scope createScopeForView(Table table, QualifiedObjectName name, Optional if (statement instanceof CreateView) { CreateView viewStatement = (CreateView) statement; QualifiedObjectName viewNameFromStatement = createQualifiedObjectName(session, viewStatement, viewStatement.getName()); + viewNameFromStatement = redirectToNewCatalogIfNecessary(session, viewNameFromStatement, metadata); if (viewStatement.isReplace() && viewNameFromStatement.equals(name)) { throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive view"); } diff --git a/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowQueriesRewrite.java b/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowQueriesRewrite.java index 00e61c5e13a4..0ef55bde9ed1 100644 --- a/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowQueriesRewrite.java +++ b/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowQueriesRewrite.java @@ -99,6 +99,7 @@ import static io.prestosql.metadata.MetadataListing.listSchemas; import static io.prestosql.metadata.MetadataUtil.createCatalogSchemaName; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.CATALOG_NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; import static io.prestosql.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY; @@ -227,6 +228,7 @@ protected Node visitShowGrants(ShowGrants showGrants, Void context) Optional tableName = showGrants.getTableName(); if (tableName.isPresent()) { QualifiedObjectName qualifiedTableName = createQualifiedObjectName(session, showGrants, tableName.get()); + qualifiedTableName = redirectToNewCatalogIfNecessary(session, qualifiedTableName, metadata); if (metadata.getView(session, qualifiedTableName).isEmpty() && metadata.getTableHandle(session, qualifiedTableName).isEmpty()) { @@ -373,6 +375,7 @@ else if (node.getLikePattern().isPresent()) { protected Node visitShowColumns(ShowColumns showColumns, Void context) { QualifiedObjectName tableName = createQualifiedObjectName(session, showColumns, showColumns.getTable()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); if (metadata.getView(session, tableName).isEmpty() && metadata.getTableHandle(session, tableName).isEmpty()) { @@ -444,6 +447,7 @@ protected Node visitShowCreate(ShowCreate node, Void context) { if (node.getType() == VIEW) { QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName()); + objectName = redirectToNewCatalogIfNecessary(session, objectName, metadata); Optional viewDefinition = metadata.getView(session, objectName); if (viewDefinition.isEmpty()) { @@ -467,6 +471,7 @@ protected Node visitShowCreate(ShowCreate node, Void context) if (node.getType() == TABLE) { QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName()); + objectName = redirectToNewCatalogIfNecessary(session, objectName, metadata); Optional viewDefinition = metadata.getView(session, objectName); if (viewDefinition.isPresent()) { @@ -483,10 +488,11 @@ protected Node visitShowCreate(ShowCreate node, Void context) Map> allColumnProperties = metadata.getColumnPropertyManager().getAllProperties().get(tableHandle.get().getCatalogName()); + QualifiedObjectName finalObjectName = objectName; List columns = connectorTableMetadata.getColumns().stream() .filter(column -> !column.isHidden()) .map(column -> { - List propertyNodes = buildProperties(objectName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties); + List propertyNodes = buildProperties(finalObjectName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties); return new ColumnDefinition(new Identifier(column.getName()), toSqlType(column.getType()), column.isNullable(), propertyNodes, Optional.ofNullable(column.getComment())); }) .collect(toImmutableList()); diff --git a/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowStatsRewrite.java b/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowStatsRewrite.java index 09a21ffeffc8..39c240ce5930 100644 --- a/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowStatsRewrite.java +++ b/presto-main/src/main/java/io/prestosql/sql/rewrite/ShowStatsRewrite.java @@ -75,6 +75,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName; +import static io.prestosql.metadata.MetadataUtil.redirectToNewCatalogIfNecessary; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; import static io.prestosql.spi.StandardErrorCode.TABLE_NOT_FOUND; import static io.prestosql.spi.type.DateType.DATE; @@ -142,6 +143,7 @@ protected Node visitShowStats(ShowStats node, Void context) Table table = getTable(node, specification); QualifiedObjectName tableName = createQualifiedObjectName(session, node, table.getName()); + tableName = redirectToNewCatalogIfNecessary(session, tableName, metadata); TableHandle tableHandle = metadata.getTableHandle(session, tableName) .orElseThrow(() -> semanticException(TABLE_NOT_FOUND, node, "Table '%s' not found", table.getName())); TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle); diff --git a/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java b/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java index e03ac81f8d4a..0867f7a1869d 100644 --- a/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java +++ b/presto-main/src/test/java/io/prestosql/metadata/AbstractMockMetadata.java @@ -749,4 +749,10 @@ public Optional> applyTopN(Session session, T { return Optional.empty(); } + + @Override + public Optional redirectCatalog(Session session, QualifiedObjectName tableName) + { + return Optional.empty(); + } } diff --git a/presto-plugin-toolkit/src/main/java/io/prestosql/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java b/presto-plugin-toolkit/src/main/java/io/prestosql/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java index 248b38eb7a74..bef0e4497c81 100644 --- a/presto-plugin-toolkit/src/main/java/io/prestosql/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/presto-plugin-toolkit/src/main/java/io/prestosql/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java @@ -740,4 +740,12 @@ public void validateScan(ConnectorSession session, ConnectorTableHandle handle) delegate.validateScan(session, handle); } } + + @Override + public Optional redirectCatalog(ConnectorSession session, SchemaTableName tableName) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.redirectCatalog(session, tableName); + } + } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveRedirectionToIceberg.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveRedirectionToIceberg.java new file mode 100644 index 000000000000..29bcf8baf445 --- /dev/null +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveRedirectionToIceberg.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.prestosql.tests.product.launcher.docker.DockerFiles; +import io.prestosql.tests.product.launcher.env.Environment; +import io.prestosql.tests.product.launcher.env.common.AbstractEnvironmentProvider; +import io.prestosql.tests.product.launcher.env.common.Hadoop; +import io.prestosql.tests.product.launcher.env.common.Standard; +import io.prestosql.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class SinglenodeHiveRedirectionToIceberg + extends AbstractEnvironmentProvider +{ + private final DockerFiles dockerFiles; + + @Inject + public SinglenodeHiveRedirectionToIceberg(DockerFiles dockerFiles, Standard standard, Hadoop hadoop) + { + super(ImmutableList.of(standard, hadoop)); + this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + } + + @Override + protected void extendEnvironment(Environment.Builder builder) + { + builder.configureContainer("presto-master", container -> container + .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-redirection-to-iceberg/hive.properties")), CONTAINER_PRESTO_HIVE_PROPERTIES) + .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-redirection-to-iceberg/iceberg.properties")), CONTAINER_PRESTO_ICEBERG_PROPERTIES)); + } +} diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/suite/suites/Suite7NonGeneric.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/suite/suites/Suite7NonGeneric.java index 8ac0a5b66707..bf4b7a379b0d 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/suite/suites/Suite7NonGeneric.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/suite/suites/Suite7NonGeneric.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import io.prestosql.tests.product.launcher.env.EnvironmentDefaults; +import io.prestosql.tests.product.launcher.env.environment.SinglenodeHiveRedirectionToIceberg; import io.prestosql.tests.product.launcher.env.environment.SinglenodeKerberosHdfsImpersonationCrossRealm; import io.prestosql.tests.product.launcher.env.environment.SinglenodeLdapBindDn; import io.prestosql.tests.product.launcher.env.environment.SinglenodeMysql; @@ -79,6 +80,14 @@ public List getTestRuns(SuiteConfig config) */ testOnEnvironment(SinglenodeSparkIceberg.class).withGroups("iceberg").withExcludedGroups("storage_formats").build(), + /** + * presto-product-tests-launcher/bin/run-launcher test run \ + * --environment singlenode-hive-redirection-to-iceberg \ + * -- -g hive_redirection_to_iceberg \ + * || suite_exit_code=1 + */ + testOnEnvironment(SinglenodeHiveRedirectionToIceberg.class).withGroups("hive_redirection_to_iceberg").build(), + /** * Environment not set up on CDH. (TODO run on HDP 2.6 and HDP 3.1) * diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-redirection-to-iceberg/hive.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-redirection-to-iceberg/hive.properties new file mode 100644 index 000000000000..96f8ca8f652f --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-redirection-to-iceberg/hive.properties @@ -0,0 +1,17 @@ +connector.name=hive-hadoop2 +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true +hive.allow-comment-table=true +hive.allow-comment-column=true +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.allow-register-partition-procedure=true +hive.metastore-cache-ttl=0s +hive.fs.cache.max-size=10 +hive.max-partitions-per-scan=100 +hive.translate-hive-views=true +hive.redirect-to-iceberg-enabled=true +hive.redirect-to-iceberg-catalog=iceberg diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-redirection-to-iceberg/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-redirection-to-iceberg/iceberg.properties new file mode 100644 index 000000000000..7ec547c05f48 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-redirection-to-iceberg/iceberg.properties @@ -0,0 +1,4 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +iceberg.file-format=PARQUET diff --git a/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java b/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java index 1c52db0d4667..8835ea461ef1 100644 --- a/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java +++ b/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java @@ -43,6 +43,7 @@ public final class TestGroups public static final String HIVE_VIEWS = "hive_views"; public static final String HIVE_CACHING = "hive_caching"; public static final String HIVE_WITH_EXTERNAL_WRITES = "hive_with_external_writes"; + public static final String HIVE_REDIRECTION_TO_ICEBERG = "hive_redirection_to_iceberg"; public static final String AUTHORIZATION = "authorization"; public static final String HIVE_COERCION = "hive_coercion"; public static final String CASSANDRA = "cassandra"; diff --git a/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveRedirectionToIceberg.java b/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveRedirectionToIceberg.java new file mode 100644 index 000000000000..e13ba70204e6 --- /dev/null +++ b/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveRedirectionToIceberg.java @@ -0,0 +1,131 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.tests.hive; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.assertj.core.api.Condition; +import org.testng.annotations.Test; + +import java.sql.Date; +import java.util.List; +import java.util.Set; + +import static io.prestosql.tempto.assertions.QueryAssert.Row; +import static io.prestosql.tempto.assertions.QueryAssert.Row.row; +import static io.prestosql.tempto.assertions.QueryAssert.assertThat; +import static io.prestosql.tempto.query.QueryExecutor.query; +import static io.prestosql.tests.TestGroups.HIVE_REDIRECTION_TO_ICEBERG; +import static io.prestosql.tests.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.prestosql.tests.utils.QueryExecutors.onPresto; +import static java.lang.String.format; + +public class TestHiveRedirectionToIceberg + extends HiveProductTest +{ + private static final String CREATE_ICEBERG_TABLE_TEMPLATE = "CREATE TABLE %s (_string VARCHAR, _bigint BIGINT, _date DATE) WITH (partitioning = ARRAY['_bigint', '_date'])"; + private static final String CREATE_HIVE_TABLE_TEMPLATE = "CREATE TABLE %s (_string VARCHAR, _bigint BIGINT, _date DATE) WITH (partitioned_by = ARRAY['_bigint', '_date'])"; + private static final String INSERT_TEMPLATE = "INSERT INTO %s VALUES " + + "(NULL, NULL, NULL), " + + "('abc', 1, DATE '2020-08-04'), " + + "('abcdefghijklmnopqrstuvwxyz', 2, DATE '2020-08-04')"; + private static final List EXPECTED_ROWS = ImmutableList.builder() + .add(row(null, null, null)) + .add(row("abc", 1, Date.valueOf("2020-08-04"))) + .add(row("abcdefghijklmnopqrstuvwxyz", 2, Date.valueOf("2020-08-04"))) + .build(); + + @Test(groups = {HIVE_REDIRECTION_TO_ICEBERG, PROFILE_SPECIFIC_TESTS}) + public void testSelectAndDrop() + { + onPresto().executeQuery(format(CREATE_ICEBERG_TABLE_TEMPLATE, "iceberg.default.test_select_drop")); + onPresto().executeQuery(format(INSERT_TEMPLATE, "iceberg.default.test_select_drop")); + assertThat(onPresto().executeQuery("SELECT * FROM iceberg.default.test_select_drop")).containsOnly(EXPECTED_ROWS); + assertThat(onPresto().executeQuery("SELECT * FROM hive.default.test_select_drop")).containsOnly(EXPECTED_ROWS); + assertThat(onPresto().executeQuery("SELECT _bigint, _date FROM hive.default.\"test_select_drop$partitions\"")).containsOnly( + row(null, null), + row(1, Date.valueOf("2020-08-04")), + row(2, Date.valueOf("2020-08-04"))); + onPresto().executeQuery("DROP TABLE hive.default.test_select_drop"); + assertThat(() -> query("SELECT * FROM hive.default.test_select_drop")) + .failsWithMessage("Table 'hive.default.test_select_drop' does not exist"); + assertThat(() -> query("SELECT * FROM iceberg.default.test_select_drop")) + .failsWithMessage("Table 'iceberg.default.test_select_drop' does not exist"); + } + + @Test(groups = {HIVE_REDIRECTION_TO_ICEBERG, PROFILE_SPECIFIC_TESTS}) + public void testInsert() + { + onPresto().executeQuery(format(CREATE_ICEBERG_TABLE_TEMPLATE, "iceberg.default.test_insert")); + onPresto().executeQuery(format(INSERT_TEMPLATE, "hive.default.test_insert")); + assertThat(onPresto().executeQuery("SELECT * FROM iceberg.default.test_insert")).containsOnly(EXPECTED_ROWS); + assertThat(onPresto().executeQuery("SELECT * FROM hive.default.test_insert")).containsOnly(EXPECTED_ROWS); + onPresto().executeQuery("DROP TABLE iceberg.default.test_insert"); + } + + @Test(groups = {HIVE_REDIRECTION_TO_ICEBERG, PROFILE_SPECIFIC_TESTS}) + public void testDescribe() + { + onPresto().executeQuery(format(CREATE_ICEBERG_TABLE_TEMPLATE, "iceberg.default.test_describe")); + assertThat(onPresto().executeQuery("DESCRIBE hive.default.test_describe")) + .satisfies(new Condition<>(queryResult -> { + Set actualColumns = ImmutableSet.copyOf(queryResult.column(1)); + Set expectedColumns = ImmutableSet.of("_string", "_bigint", "_date"); + return actualColumns.equals(expectedColumns); + }, "equals")); + onPresto().executeQuery("DROP TABLE iceberg.default.test_describe"); + } + + @Test(groups = {HIVE_REDIRECTION_TO_ICEBERG, PROFILE_SPECIFIC_TESTS}) + public void testShowCreateTable() + { + onPresto().executeQuery(format(CREATE_ICEBERG_TABLE_TEMPLATE, "iceberg.default.test_show_create")); + assertThat(onPresto().executeQuery("SHOW CREATE TABLE hive.default.test_show_create")).hasRowsCount(1); + onPresto().executeQuery("DROP TABLE iceberg.default.test_show_create"); + } + + @Test(groups = {HIVE_REDIRECTION_TO_ICEBERG, PROFILE_SPECIFIC_TESTS}) + public void testAlterTable() + { + onPresto().executeQuery(format(CREATE_ICEBERG_TABLE_TEMPLATE, "iceberg.default.test_alter_table")); + onPresto().executeQuery(format(INSERT_TEMPLATE, "iceberg.default.test_alter_table")); + + onPresto().executeQuery("ALTER TABLE hive.default.test_alter_table RENAME TO default.test_alter_table_new"); + assertThat(onPresto().executeQuery("SELECT * FROM iceberg.default.test_alter_table_new")).containsOnly(EXPECTED_ROWS); + assertThat(onPresto().executeQuery("SELECT * FROM hive.default.test_alter_table_new")).containsOnly(EXPECTED_ROWS); + + onPresto().executeQuery("ALTER TABLE hive.default.test_alter_table_new ADD COLUMN _double DOUBLE"); + onPresto().executeQuery("ALTER TABLE hive.default.test_alter_table_new DROP COLUMN _string"); + onPresto().executeQuery("ALTER TABLE hive.default.test_alter_table_new RENAME COLUMN _bigint TO _bi"); + assertThat(onPresto().executeQuery("DESCRIBE hive.default.test_alter_table_new")) + .satisfies(new Condition<>(queryResult -> { + Set actualColumns = ImmutableSet.copyOf(queryResult.column(1)); + Set expectedColumns = ImmutableSet.of("_bi", "_date", "_double"); + return actualColumns.equals(expectedColumns); + }, "equals")); + + onPresto().executeQuery("DROP TABLE iceberg.default.test_alter_table_new"); + } + + @Test(groups = {HIVE_REDIRECTION_TO_ICEBERG, PROFILE_SPECIFIC_TESTS}) + public void testCreateHiveTable() + { + onPresto().executeQuery(format(CREATE_HIVE_TABLE_TEMPLATE, "hive.default.test_create_hive_table")); + onPresto().executeQuery(format(INSERT_TEMPLATE, "hive.default.test_create_hive_table")); + assertThat(onPresto().executeQuery("SELECT * FROM hive.default.test_create_hive_table")).containsOnly(EXPECTED_ROWS); + assertThat(() -> query("SELECT * FROM iceberg.default.test_create_hive_table")) + .failsWithMessage("Not an Iceberg table: default.test_create_hive_table"); + onPresto().executeQuery("DROP TABLE hive.default.test_create_hive_table"); + } +} diff --git a/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorMetadata.java b/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorMetadata.java index 4081b899e795..bd04e2599bbc 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorMetadata.java +++ b/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorMetadata.java @@ -965,4 +965,12 @@ default Optional> applyTopN( *

*/ default void validateScan(ConnectorSession session, ConnectorTableHandle handle) {} + + /** + * Redirects to another catalog which may or may not use the same connector + */ + default Optional redirectCatalog(ConnectorSession session, SchemaTableName tableName) + { + return Optional.empty(); + } }