diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MaterializedViewAlreadyExistsException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MaterializedViewAlreadyExistsException.java new file mode 100644 index 0000000000000..6e1c31c5fbfb6 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MaterializedViewAlreadyExistsException.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaTableName; + +import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; +import static java.lang.String.format; + +public class MaterializedViewAlreadyExistsException + extends PrestoException +{ + private final SchemaTableName viewName; + + public MaterializedViewAlreadyExistsException(SchemaTableName viewName) + { + this(viewName, format("Materialized view already exists: '%s'", viewName)); + } + + public MaterializedViewAlreadyExistsException(SchemaTableName viewName, String message) + { + this(viewName, message, null); + } + + public MaterializedViewAlreadyExistsException(SchemaTableName viewName, String message, Throwable cause) + { + super(ALREADY_EXISTS, message, cause); + this.viewName = viewName; + } + + public SchemaTableName getViewName() + { + return viewName; + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java index 42df84a754fce..6eb4c138ea591 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java @@ -48,6 +48,7 @@ import com.facebook.presto.spi.statistics.ColumnStatisticType; import com.google.common.base.CharMatcher; import com.google.common.base.Joiner; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -102,6 +103,7 @@ import static com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_TRUE_VALUES; import static com.facebook.presto.spi.statistics.ColumnStatisticType.TOTAL_SIZE_IN_BYTES; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Strings.padEnd; import static com.google.common.io.BaseEncoding.base16; @@ -131,6 +133,7 @@ public class MetastoreUtil public static final String PRESTO_OFFLINE = "presto_offline"; public static final String AVRO_SCHEMA_URL_KEY = "avro.schema.url"; public static final String PRESTO_VIEW_FLAG = "presto_view"; + public static final String PRESTO_MATERIALIZED_VIEW_FLAG = "presto_materialized_view"; public static final String PRESTO_QUERY_ID_NAME = "presto_query_id"; public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; @SuppressWarnings("OctalInteger") @@ -654,6 +657,17 @@ public static boolean isPrestoView(Table table) return "true".equals(table.getParameters().get(PRESTO_VIEW_FLAG)); } + public static boolean isPrestoMaterializedView(Table table) + { + if ("true".equals(table.getParameters().get(PRESTO_MATERIALIZED_VIEW_FLAG))) { + checkState( + !table.getViewOriginalText().map(Strings::isNullOrEmpty).orElse(true), + "viewOriginalText field is not set for the Table metadata of materialized view %s.", table.getTableName()); + return true; + } + return false; + } + private static String getRenameErrorMessage(Path source, Path target) { return format("Error moving data files from %s to final location %s", source, target); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java index 4bc526da684a4..4934332400e13 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java @@ -20,6 +20,7 @@ public enum PrestoTableType MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, + MATERIALIZED_VIEW, TEMPORARY_TABLE, OTHER, // Some Hive implementations define additional table types /**/; diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java index fbc0bc977351b..2806b2ebd25f6 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java @@ -77,6 +77,7 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.renameFile; import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; +import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.SUBTRACT; @@ -491,7 +492,7 @@ public synchronized void truncateUnpartitionedTable(ConnectorSession session, St if (!table.isPresent()) { throw new TableNotFoundException(schemaTableName); } - if (!table.get().getTableType().equals(MANAGED_TABLE)) { + if (!table.get().getTableType().equals(MANAGED_TABLE) && !table.get().getTableType().equals(MATERIALIZED_VIEW)) { throw new PrestoException(NOT_SUPPORTED, "Cannot delete from non-managed Hive table"); } if (!table.get().getPartitionColumns().isEmpty()) { @@ -1132,7 +1133,7 @@ private void prepareAddTable(HdfsContext context, TableAndMore tableAndMore) return; } - if (table.getTableType().equals(MANAGED_TABLE)) { + if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) { String targetLocation = table.getStorage().getLocation(); checkArgument(!targetLocation.isEmpty(), "target location is empty"); Optional currentPath = tableAndMore.getCurrentLocation(); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java index 4e0ed831c880f..3411ea82a119c 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java @@ -87,6 +87,7 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; +import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; @@ -235,7 +236,7 @@ public synchronized void createTable(Table table, PrincipalPrivileges principalP if (table.getTableType().equals(VIRTUAL_VIEW)) { checkArgument(table.getStorage().getLocation().isEmpty(), "Storage location for view must be empty"); } - else if (table.getTableType().equals(MANAGED_TABLE)) { + else if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) { if (!tableMetadataDirectory.equals(new Path(table.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory must be " + tableMetadataDirectory); } @@ -412,7 +413,7 @@ public synchronized void dropTable(String databaseName, String tableName, boolea Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName); // It is safe to delete the whole meta directory for external tables and views - if (!table.getTableType().equals(MANAGED_TABLE) || deleteData) { + if ((!table.getTableType().equals(MANAGED_TABLE) && !table.getTableType().equals(MATERIALIZED_VIEW)) || deleteData) { deleteMetadataDirectory(tableMetadataDirectory); } else { @@ -566,7 +567,7 @@ public synchronized void addPartitions(String databaseName, String tableName, Li Table table = getRequiredTable(databaseName, tableName); - checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(table.getTableType()), "Invalid table type: %s", table.getTableType()); + checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, MATERIALIZED_VIEW).contains(table.getTableType()), "Invalid table type: %s", table.getTableType()); try { Map schemaFiles = new LinkedHashMap<>(); @@ -614,7 +615,7 @@ private void verifiedPartition(Table table, Partition partition) { Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); - if (table.getTableType().equals(MANAGED_TABLE)) { + if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) { if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory); } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java index 45f096ebb94ca..5d64b3665b94f 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java @@ -40,6 +40,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.primitives.Shorts; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -99,14 +100,17 @@ import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE; import static com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY; +import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_MATERIALIZED_VIEW_FLAG; import static com.facebook.presto.hive.metastore.MetastoreUtil.fromMetastoreDistinctValuesCount; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; +import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW; import static com.facebook.presto.hive.metastore.PrestoTableType.OTHER; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; import static com.facebook.presto.spi.security.PrincipalType.ROLE; import static com.facebook.presto.spi.security.PrincipalType.USER; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableSet.toImmutableSet; @@ -146,11 +150,20 @@ public static org.apache.hadoop.hive.metastore.api.Database toMetastoreApiDataba public static org.apache.hadoop.hive.metastore.api.Table toMetastoreApiTable(Table table, PrincipalPrivileges privileges) { org.apache.hadoop.hive.metastore.api.Table result = new org.apache.hadoop.hive.metastore.api.Table(); + result.setDbName(table.getDatabaseName()); result.setTableName(table.getTableName()); result.setOwner(table.getOwner()); - checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW).contains(table.getTableType()), "Invalid table type: %s", table.getTableType()); - result.setTableType(table.getTableType().name()); + + PrestoTableType tableType = table.getTableType(); + checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, MATERIALIZED_VIEW).contains(tableType), "Invalid table type: %s", table.getTableType()); + // TODO: remove the table type change after Hive 3.0 upgrade. + // TableType.MATERIALIZED_VIEW is not supported by Hive metastore until Hive 3.0. Use MANAGED_TABLE for now. + if (MATERIALIZED_VIEW.equals(tableType)) { + tableType = MANAGED_TABLE; + } + result.setTableType(tableType.name()); + result.setParameters(table.getParameters()); result.setPartitionKeys(table.getPartitionColumns().stream().map(ThriftMetastoreUtil::toMetastoreApiFieldSchema).collect(toList())); result.setSd(makeStorageDescriptor(table.getTableName(), table.getDataColumns(), table.getStorage())); @@ -400,11 +413,19 @@ public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.T throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor"); } + // TODO: remove the table type change after Hive 3.0 update. + // Materialized view fetched from Hive Metastore uses TableType.MANAGED_TABLE before Hive 3.0. Cast it back to MATERIALIZED_VIEW here. + PrestoTableType tableType = PrestoTableType.optionalValueOf(table.getTableType()).orElse(OTHER); + if (table.getParameters() != null && "true".equals(table.getParameters().get(PRESTO_MATERIALIZED_VIEW_FLAG))) { + checkState(TableType.MANAGED_TABLE.name().equals(table.getTableType()), "Materialized view %s has incorrect table type %s from Metastore.", table.getTableName(), table.getTableType()); + tableType = MATERIALIZED_VIEW; + } + Table.Builder tableBuilder = Table.builder() .setDatabaseName(table.getDbName()) .setTableName(table.getTableName()) .setOwner(nullToEmpty(table.getOwner())) - .setTableType(PrestoTableType.optionalValueOf(table.getTableType()).orElse(OTHER)) + .setTableType(tableType) .setDataColumns(schema.stream() .map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema) .collect(toList())) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 0f2e81ca894f9..973181ea3296c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -46,6 +46,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorMetadataUpdateHandle; import com.facebook.presto.spi.ConnectorNewTableLayout; import com.facebook.presto.spi.ConnectorOutputTableHandle; @@ -60,6 +61,7 @@ import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.DiscretePredicates; import com.facebook.presto.spi.InMemoryRecordSet; +import com.facebook.presto.spi.MaterializedViewNotFoundException; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.RecordCursor; @@ -143,6 +145,7 @@ import java.util.stream.Stream; import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.common.predicate.TupleDomain.withColumnDomains; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -245,11 +248,14 @@ import static com.facebook.presto.hive.HiveTableProperties.getOrcBloomFilterFpp; import static com.facebook.presto.hive.HiveTableProperties.getPartitionedBy; import static com.facebook.presto.hive.HiveTableProperties.getPreferredOrderingColumns; +import static com.facebook.presto.hive.HiveTableProperties.isExternalTable; import static com.facebook.presto.hive.HiveType.HIVE_BINARY; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveType.toHiveType; import static com.facebook.presto.hive.HiveUtil.columnExtraInfo; +import static com.facebook.presto.hive.HiveUtil.decodeMaterializedViewData; import static com.facebook.presto.hive.HiveUtil.decodeViewData; +import static com.facebook.presto.hive.HiveUtil.encodeMaterializedViewData; import static com.facebook.presto.hive.HiveUtil.encodeViewData; import static com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles; import static com.facebook.presto.hive.HiveUtil.hiveColumnHandles; @@ -265,6 +271,7 @@ import static com.facebook.presto.hive.PartitionUpdate.UpdateMode.OVERWRITE; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.toHivePrivilege; import static com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY; +import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_MATERIALIZED_VIEW_FLAG; import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME; import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; @@ -273,6 +280,7 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; +import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.ADD; @@ -289,6 +297,7 @@ import static com.facebook.presto.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY; import static com.facebook.presto.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY; import static com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_VIEW; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY; import static com.facebook.presto.spi.TableLayoutFilterCoverage.COVERED; @@ -315,6 +324,7 @@ import static com.google.common.collect.Streams.stream; import static java.lang.String.format; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; @@ -349,6 +359,8 @@ public class HiveMetadata private static final String CSV_QUOTE_KEY = OpenCSVSerde.QUOTECHAR; private static final String CSV_ESCAPE_KEY = OpenCSVSerde.ESCAPECHAR; + private static final JsonCodec MATERIALIZED_VIEW_JSON_CODEC = jsonCodec(ConnectorMaterializedViewDefinition.class); + private final boolean allowCorruptWritesForTesting; private final SemiTransactionalHiveMetastore metastore; private final HdfsEnvironment hdfsEnvironment; @@ -891,6 +903,22 @@ public void renameSchema(ConnectorSession session, String source, String target) @Override public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) + { + PrestoTableType tableType = isExternalTable(tableMetadata.getProperties()) ? EXTERNAL_TABLE : MANAGED_TABLE; + Table table = prepareTable(session, tableMetadata, tableType); + PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(table.getOwner()); + HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? createZeroStatistics() : createEmptyStatistics(); + + metastore.createTable( + session, + table, + principalPrivileges, + Optional.empty(), + ignoreExisting, + new PartitionStatistics(basicStatistics, ImmutableMap.of())); + } + + private Table prepareTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, PrestoTableType tableType) { SchemaTableName schemaTableName = tableMetadata.getTable(); String schemaName = schemaTableName.getSchemaName(); @@ -923,21 +951,20 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe checkPartitionTypesSupported(partitionColumns); Path targetPath; - PrestoTableType tableType; - String externalLocation = getExternalLocation(tableMetadata.getProperties()); - if (externalLocation != null) { + if (tableType.equals(EXTERNAL_TABLE)) { if (!createsOfNonManagedTablesEnabled) { throw new PrestoException(NOT_SUPPORTED, "Cannot create non-managed Hive table"); } - - tableType = EXTERNAL_TABLE; + String externalLocation = getExternalLocation(tableMetadata.getProperties()); targetPath = getExternalPath(new HdfsContext(session, schemaName, tableName, externalLocation, true), externalLocation); } - else { - tableType = MANAGED_TABLE; + else if (tableType.equals(MANAGED_TABLE) || tableType.equals(MATERIALIZED_VIEW)) { LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty, preferredOrderingColumns)); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); } + else { + throw new IllegalStateException(format("%s is not a valid table type to be created.", tableType)); + } Map tableProperties = getEmptyTableProperties( tableMetadata, @@ -945,7 +972,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe hiveStorageFormat, tableEncryptionProperties); - Table table = buildTableObject( + return buildTableObject( session.getQueryId(), schemaName, tableName, @@ -959,15 +986,6 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe targetPath, tableType, prestoVersion); - PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(table.getOwner()); - HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? createZeroStatistics() : createEmptyStatistics(); - metastore.createTable( - session, - table, - principalPrivileges, - Optional.empty(), - ignoreExisting, - new PartitionStatistics(basicStatistics, ImmutableMap.of())); } @Override @@ -2228,6 +2246,84 @@ public Map getViews(ConnectorSession s return views.build(); } + @Override + public Optional getMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + requireNonNull(viewName, "viewName is null"); + + Optional table = metastore.getTable(viewName.getSchemaName(), viewName.getTableName()); + + if (table.isPresent() && MetastoreUtil.isPrestoMaterializedView(table.get())) { + try { + return Optional.of(MATERIALIZED_VIEW_JSON_CODEC.fromJson(decodeMaterializedViewData(table.get().getViewOriginalText().get()))); + } + catch (IllegalArgumentException e) { + throw new PrestoException(INVALID_VIEW, "Invalid materialized view JSON", e); + } + } + + return Optional.empty(); + } + + @Override + public void createMaterializedView(ConnectorSession session, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + if (isExternalTable(viewMetadata.getProperties())) { + throw new PrestoException(INVALID_TABLE_PROPERTY, "Specifying external location for materialized view is not supported."); + } + + Table basicTable = prepareTable(session, viewMetadata, MATERIALIZED_VIEW); + Map parameters = ImmutableMap.builder() + .putAll(basicTable.getParameters()) + .put(PRESTO_MATERIALIZED_VIEW_FLAG, "true") + .build(); + Table viewTable = Table.builder(basicTable) + .setParameters(parameters) + .setViewOriginalText(Optional.of(encodeMaterializedViewData(MATERIALIZED_VIEW_JSON_CODEC.toJson(viewDefinition)))) + .setViewExpandedText(Optional.of("/* Presto Materialized View */")) + .build(); + + List
baseTables = viewDefinition.getBaseTables().stream() + .map(baseTableName -> metastore.getTable(baseTableName.getSchemaName(), baseTableName.getTableName()) + .orElseThrow(() -> new TableNotFoundException(baseTableName))) + .collect(toImmutableList()); + + validateMaterializedViewPartitionColumns(viewTable, baseTables, viewDefinition.getColumnMappingsAsMap()); + + try { + PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(viewTable.getOwner()); + metastore.createTable( + session, + viewTable, + principalPrivileges, + Optional.empty(), + ignoreExisting, + new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); + } + catch (TableAlreadyExistsException e) { + throw new MaterializedViewAlreadyExistsException(e.getTableName()); + } + } + + @Override + public void dropMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + Optional view = getMaterializedView(session, viewName); + if (!view.isPresent()) { + throw new MaterializedViewNotFoundException(viewName); + } + + try { + metastore.dropTable( + new HdfsContext(session, viewName.getSchemaName(), viewName.getTableName()), + viewName.getSchemaName(), + viewName.getTableName()); + } + catch (TableNotFoundException e) { + throw new MaterializedViewNotFoundException(e.getTableName()); + } + } + @Override public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTableHandle tableHandle) { @@ -3160,6 +3256,54 @@ private static void validatePartitionColumns(ConnectorTableMetadata tableMetadat } } + /** + * Validate the partition columns of a materialized view to ensure 1) a materialized view is partitioned; and 2) it has at least one partition + * directly mapped to a base table partition. + * + * A column is directly mapped to a base table column if it is derived directly or transitively from the base table column, + * by only selecting a column or an aliased column without any function or operator applied. + * For example, with SELECT column_b AS column_a, column_a is directly mapped to column_b. + * With SELECT column_b + column_c AS column_a, column_a is not directly mapped to any column. + * + * {@code viewToBaseColumnMap} only contains direct column mappings. + */ + private static void validateMaterializedViewPartitionColumns(Table viewTable, List
baseTables, Map> viewToBaseColumnMap) + { + SchemaTableName viewName = new SchemaTableName(viewTable.getDatabaseName(), viewTable.getTableName()); + if (viewToBaseColumnMap.isEmpty()) { + throw new PrestoException( + NOT_SUPPORTED, + format("Materialized view %s must have at least one column directly defined by a base table column.", viewName.toString())); + } + + List viewPartitions = viewTable.getPartitionColumns(); + if (viewPartitions.isEmpty()) { + throw new PrestoException(NOT_SUPPORTED, "Unpartitioned materialized view is not supported."); + } + + Map> baseTablePartitions = baseTables.stream() + .collect(toImmutableMap( + table -> new SchemaTableName(table.getDatabaseName(), table.getTableName()), + Table::getPartitionColumns)); + + for (Column viewPartition : viewPartitions) { + for (SchemaTableName baseTable : baseTablePartitions.keySet()) { + for (Column basePartition : baseTablePartitions.get(baseTable)) { + if (viewToBaseColumnMap + .getOrDefault(viewPartition.getName(), emptyMap()) + .getOrDefault(baseTable, "") + .equals(basePartition.getName())) { + return; + } + } + } + } + + throw new PrestoException( + NOT_SUPPORTED, + format("Materialized view %s must have at least one partition directly defined by a base table partition.", viewName.toString())); + } + protected Optional getTableEncryptionPropertiesFromTableProperties(ConnectorTableMetadata tableMetadata, HiveStorageFormat hiveStorageFormat, List partitionedBy) { ColumnEncryptionInformation columnEncryptionInformation = getEncryptColumns(tableMetadata.getProperties()); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java index 53fe58b7de990..88b5d9bb88bf6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java @@ -177,6 +177,11 @@ public static String getExternalLocation(Map tableProperties) return (String) tableProperties.get(EXTERNAL_LOCATION_PROPERTY); } + public static boolean isExternalTable(Map tableProperties) + { + return tableProperties.get(EXTERNAL_LOCATION_PROPERTY) != null; + } + public static String getAvroSchemaUrl(Map tableProperties) { return (String) tableProperties.get(AVRO_SCHEMA_URL); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 6be130bc9ab14..041cc10bddad8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -175,6 +175,8 @@ public final class HiveUtil private static final String VIEW_PREFIX = "/* Presto View: "; private static final String VIEW_SUFFIX = " */"; + private static final String MATERIALIZED_VIEW_PREFIX = "/* Presto Materialized View: "; + private static final String MATERIALIZED_VIEW_SUFFIX = " */"; private static final DateTimeFormatter HIVE_DATE_PARSER = ISODateTimeFormat.date().withZoneUTC(); private static final DateTimeFormatter HIVE_TIMESTAMP_PARSER; @@ -652,15 +654,35 @@ public static NullableValue parsePartitionValue(String partitionName, String val public static String encodeViewData(String data) { - return VIEW_PREFIX + Base64.getEncoder().encodeToString(data.getBytes(UTF_8)) + VIEW_SUFFIX; + return encodeView(data, VIEW_PREFIX, VIEW_SUFFIX); } public static String decodeViewData(String data) { - checkCondition(data.startsWith(VIEW_PREFIX), HIVE_INVALID_VIEW_DATA, "View data missing prefix: %s", data); - checkCondition(data.endsWith(VIEW_SUFFIX), HIVE_INVALID_VIEW_DATA, "View data missing suffix: %s", data); - data = data.substring(VIEW_PREFIX.length()); - data = data.substring(0, data.length() - VIEW_SUFFIX.length()); + return decodeView(data, VIEW_PREFIX, VIEW_SUFFIX); + } + + public static String encodeMaterializedViewData(String data) + { + return encodeView(data, MATERIALIZED_VIEW_PREFIX, MATERIALIZED_VIEW_SUFFIX); + } + + public static String decodeMaterializedViewData(String data) + { + return decodeView(data, MATERIALIZED_VIEW_PREFIX, MATERIALIZED_VIEW_SUFFIX); + } + + private static String encodeView(String data, String prefix, String suffix) + { + return prefix + Base64.getEncoder().encodeToString(data.getBytes(UTF_8)) + suffix; + } + + private static String decodeView(String data, String prefix, String suffix) + { + checkCondition(data.startsWith(prefix), HIVE_INVALID_VIEW_DATA, "View data missing prefix: %s", data); + checkCondition(data.endsWith(suffix), HIVE_INVALID_VIEW_DATA, "View data missing suffix: %s", data); + data = data.substring(prefix.length()); + data = data.substring(0, data.length() - suffix.length()); return new String(Base64.getDecoder().decode(data), UTF_8); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index 243a7b4bb0f83..abb30142f0dcf 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -113,6 +113,7 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.pathExists; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; +import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static java.lang.Float.intBitsToFloat; @@ -299,6 +300,7 @@ public static void checkTableIsWritable(Table table, boolean writesToNonManagedT PrestoTableType tableType = table.getTableType(); if (!writesToNonManagedTablesEnabled && !tableType.equals(MANAGED_TABLE) + && !tableType.equals(MATERIALIZED_VIEW) && !tableType.equals(TEMPORARY_TABLE)) { throw new PrestoException(NOT_SUPPORTED, "Cannot write to non-managed Hive table"); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index 24d57a4189ee9..25234d867f0d4 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -5301,6 +5301,139 @@ public void testUnsupportedCsvTable() "\\QHive CSV storage format only supports VARCHAR (unbounded). Unsupported columns: i integer, bound varchar(10)\\E"); } + @Test + public void testCreateMaterializedView() + { + computeActual("CREATE TABLE test_customer_base WITH (partitioned_by = ARRAY['nationkey']) AS SELECT custkey, name, address, nationkey FROM customer LIMIT 10"); + computeActual("CREATE TABLE test_orders_base WITH (partitioned_by = ARRAY['orderstatus']) AS SELECT orderkey, custkey, totalprice, orderstatus FROM orders LIMIT 10"); + + // Test successful create + assertUpdate("CREATE MATERIALIZED VIEW test_customer_mv WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM test_customer_base"); + assertTrue(getQueryRunner().tableExists(getSession(), "test_customer_mv")); + assertTableColumnNames("test_customer_mv", "name", "nationkey"); + + // Test if exists + assertQueryFails( + "CREATE MATERIALIZED VIEW test_customer_mv AS SELECT name FROM test_customer_base", + format(".* Materialized view '%s.%s.test_customer_mv' already exists", getSession().getCatalog().get(), getSession().getSchema().get())); + assertQuerySucceeds("CREATE MATERIALIZED VIEW IF NOT EXISTS test_customer_mv AS SELECT name FROM test_customer_base"); + + // Test partition mapping + assertQueryFails( + "CREATE MATERIALIZED VIEW test_customer_mv_no_partition " + withRetentionDays(30) + " AS SELECT name FROM test_customer_base", + ".*Unpartitioned materialized view is not supported."); + assertQueryFails( + "CREATE MATERIALIZED VIEW test_customer_mv_no_direct_partition_mapped WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, CAST(nationkey AS BIGINT) AS nationkey FROM test_customer_base", + format(".*Materialized view %s.test_customer_mv_no_direct_partition_mapped must have at least one partition directly defined by a base table partition.", getSession().getSchema().get())); + + // Test nested + assertQueryFails( + "CREATE MATERIALIZED VIEW test_customer_nested_mv WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM test_customer_mv", + format(".*CreateMaterializedView on a materialized view %s.%s.test_customer_mv is not supported.", getSession().getCatalog().get(), getSession().getSchema().get())); + + // Test query shape + assertUpdate("CREATE MATERIALIZED VIEW test_customer_agg_mv WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT COUNT(DISTINCT name) AS num, nationkey FROM (SELECT name, nationkey FROM test_customer_base GROUP BY 1, 2) a GROUP BY nationkey"); + assertUpdate("CREATE MATERIALIZED VIEW test_customer_union_mv WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM ( SELECT name, nationkey FROM test_customer_base WHERE nationkey = 1 UNION ALL SELECT name, nationkey FROM test_customer_base WHERE nationkey = 2)"); + assertUpdate("CREATE MATERIALIZED VIEW test_customer_order_join_mv WITH (partitioned_by = ARRAY['orderstatus', 'nationkey']" + retentionDays(30) + ") AS SELECT orders.totalprice, orders.orderstatus, customer.nationkey FROM test_customer_base customer JOIN test_orders_base orders ON orders.custkey = customer.custkey"); + assertQueryFails( + "CREATE MATERIALIZED VIEW test_customer_order_join_mv_no_base_partition_mapped WITH (partitioned_by = ARRAY['custkey']" + retentionDays(30) + ") AS SELECT orders.totalprice, customer.nationkey, customer.custkey FROM test_customer_base customer JOIN test_orders_base orders ON orders.custkey = customer.custkey", + format(".*Materialized view %s.test_customer_order_join_mv_no_base_partition_mapped must have at least one partition directly defined by a base table partition.", getSession().getSchema().get())); + assertQueryFails( + "CREATE MATERIALIZED VIEW test_customer_order_join_mv_no_base_partition_mapped WITH (partitioned_by = ARRAY['nation_order']" + retentionDays(30) + ") AS SELECT orders.totalprice, CONCAT(CAST(customer.nationkey AS VARCHAR), orders.orderstatus) AS nation_order FROM test_customer_base customer JOIN test_orders_base orders ON orders.custkey = customer.custkey", + format(".*Materialized view %s.test_customer_order_join_mv_no_base_partition_mapped must have at least one partition directly defined by a base table partition.", getSession().getSchema().get())); + + // Clean up + computeActual("DROP TABLE IF EXISTS test_customer_base"); + computeActual("DROP TABLE IF EXISTS test_orders_base"); + } + + @Test + public void testShowCreateOnMaterializedView() + { + computeActual("CREATE TABLE test_customer_base_1 WITH (partitioned_by = ARRAY['nationkey']) AS SELECT custkey, name, address, nationkey FROM customer LIMIT 10"); + computeActual("CREATE MATERIALIZED VIEW test_customer_mv_1 WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM test_customer_base_1"); + + assertQueryFails( + "SHOW CREATE VIEW test_customer_mv_1", + format(".*Relation '%s.%s.test_customer_mv_1' is a materialized view, not a view", getSession().getCatalog().get(), getSession().getSchema().get())); + assertQueryFails( + "SHOW CREATE TABLE test_customer_mv_1", + format(".*Relation '%s.%s.test_customer_mv_1' is a materialized view, not a table", getSession().getCatalog().get(), getSession().getSchema().get())); + + // Clean up + computeActual("DROP TABLE IF EXISTS test_customer_base_1"); + } + + @Test + public void testAlterOnMaterializedView() + { + computeActual("CREATE TABLE test_customer_base_2 WITH (partitioned_by = ARRAY['nationkey']) AS SELECT custkey, name, address, nationkey FROM customer LIMIT 10"); + computeActual("CREATE MATERIALIZED VIEW test_customer_mv_2 WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM test_customer_base_2"); + + assertQueryFails( + "ALTER TABLE test_customer_mv_2 RENAME TO test_customer_mv_new", + format(".*'%s.%s.test_customer_mv_2' is a materialized view, and rename is not supported", getSession().getCatalog().get(), getSession().getSchema().get())); + assertQueryFails( + "ALTER TABLE test_customer_mv_2 ADD COLUMN timezone VARCHAR", + format(".*'%s.%s.test_customer_mv_2' is a materialized view, and add column is not supported", getSession().getCatalog().get(), getSession().getSchema().get())); + assertQueryFails( + "ALTER TABLE test_customer_mv_2 DROP COLUMN address", + format(".*'%s.%s.test_customer_mv_2' is a materialized view, and drop column is not supported", getSession().getCatalog().get(), getSession().getSchema().get())); + assertQueryFails( + "ALTER TABLE test_customer_mv_2 RENAME COLUMN name TO custname", + format(".*'%s.%s.test_customer_mv_2' is a materialized view, and rename column is not supported", getSession().getCatalog().get(), getSession().getSchema().get())); + + // Clean up + computeActual("DROP TABLE IF EXISTS test_customer_base_2"); + } + + @Test + public void testInsertDeleteOnMaterializedView() + { + computeActual("CREATE TABLE test_customer_base_3 WITH (partitioned_by = ARRAY['nationkey']) AS SELECT custkey, name, address, nationkey FROM customer LIMIT 10"); + computeActual("CREATE MATERIALIZED VIEW test_customer_mv_3 WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM test_customer_base_3"); + + assertQueryFails( + "INSERT INTO test_customer_mv_3 SELECT name, nationkey FROM test_customer_base_2", + ".*Inserting into materialized views is not supported"); + assertQueryFails( + "DELETE FROM test_customer_mv_3", + ".*Deleting from materialized views is not supported"); + + // Clean up + computeActual("DROP TABLE IF EXISTS test_customer_base_3"); + } + + @Test + public void testDropMaterializedView() + { + computeActual("CREATE TABLE test_customer_base_4 WITH (partitioned_by = ARRAY['nationkey']) AS SELECT custkey, name, address, nationkey FROM customer LIMIT 10"); + computeActual("CREATE MATERIALIZED VIEW test_customer_mv_4 WITH (partitioned_by = ARRAY['nationkey']" + retentionDays(30) + ") AS SELECT name, nationkey FROM test_customer_base_4"); + + assertQueryFails( + "DROP TABLE test_customer_mv_4", + format(".*'%s.%s.test_customer_mv_4' is a materialized view, not a table. Use DROP MATERIALIZED VIEW to drop.", getSession().getCatalog().get(), getSession().getSchema().get())); + assertQueryFails( + "DROP VIEW test_customer_mv_4", + format(".*View '%s.%s.test_customer_mv_4' does not exist", getSession().getCatalog().get(), getSession().getSchema().get())); + + assertUpdate("DROP MATERIALIZED VIEW test_customer_mv_4"); + assertFalse(getQueryRunner().tableExists(getSession(), "test_customer_mv_4")); + + // Clean up + computeActual("DROP TABLE IF EXISTS test_customer_base_4"); + } + + protected String retentionDays(int days) + { + return ""; + } + + protected String withRetentionDays(int days) + { + return ""; + } + private Session getParallelWriteSession() { return Session.builder(getSession()) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/AddColumnTask.java b/presto-main/src/main/java/com/facebook/presto/execution/AddColumnTask.java index 4c919e8800bd4..5a5ce2fae3d28 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/AddColumnTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/AddColumnTask.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.sql.analyzer.SemanticException; @@ -69,6 +70,14 @@ public ListenableFuture execute(AddColumn statement, TransactionManager trans return immediateFuture(null); } + Optional optionalMaterializedView = metadata.getMaterializedView(session, tableName); + if (optionalMaterializedView.isPresent()) { + if (!statement.isTableExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and add column is not supported", tableName); + } + return immediateFuture(null); + } + ConnectorId connectorId = metadata.getCatalogHandle(session, tableName.getCatalogName()) .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName())); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java b/presto-main/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java new file mode 100644 index 0000000000000..e837f0bb6b468 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.sql.analyzer.Analysis; +import com.facebook.presto.sql.analyzer.Analyzer; +import com.facebook.presto.sql.analyzer.SemanticException; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.CreateMaterializedView; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.util.concurrent.ListenableFuture; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; +import static com.facebook.presto.metadata.MetadataUtil.toSchemaTableName; +import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; +import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; +import static com.facebook.presto.sql.NodeUtils.mapFromProperties; +import static com.facebook.presto.sql.SqlFormatterUtil.getFormattedSql; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MATERIALIZED_VIEW_ALREADY_EXISTS; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.util.Objects.requireNonNull; + +public class CreateMaterializedViewTask + implements DataDefinitionTask +{ + private final SqlParser sqlParser; + + @Inject + public CreateMaterializedViewTask(SqlParser sqlParser) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + @Override + public String getName() + { + return "CREATE MATERIALIZED VIEW"; + } + + @Override + public ListenableFuture execute(CreateMaterializedView statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) + { + Session session = stateMachine.getSession(); + QualifiedObjectName viewName = createQualifiedObjectName(session, statement, statement.getName()); + + Optional viewHandle = metadata.getTableHandle(session, viewName); + if (viewHandle.isPresent()) { + if (!statement.isNotExists()) { + throw new SemanticException(MATERIALIZED_VIEW_ALREADY_EXISTS, statement, "Materialized view '%s' already exists", viewName); + } + return immediateFuture(null); + } + + accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), viewName); + accessControl.checkCanCreateView(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), viewName); + + Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, stateMachine.getWarningCollector()); + Analysis analysis = analyzer.analyze(statement); + + ConnectorId connectorId = metadata.getCatalogHandle(session, viewName.getCatalogName()) + .orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + viewName.getCatalogName())); + List columnMetadata = analysis.getOutputDescriptor(statement.getQuery()) + .getVisibleFields().stream() + .map(field -> new ColumnMetadata(field.getName().get(), field.getType())) + .collect(toImmutableList()); + + Map sqlProperties = mapFromProperties(statement.getProperties()); + Map properties = metadata.getTablePropertyManager().getProperties( + connectorId, + viewName.getCatalogName(), + sqlProperties, + session, + metadata, + parameters); + + ConnectorTableMetadata viewMetadata = new ConnectorTableMetadata( + toSchemaTableName(viewName), + columnMetadata, + properties, + statement.getComment()); + + String sql = getFormattedSql(statement.getQuery(), sqlParser, Optional.of(parameters)); + + List baseTables = analysis.getTableNodes().stream() + .map(table -> { + QualifiedObjectName tableName = createQualifiedObjectName(session, table, table.getName()); + if (!viewName.getCatalogName().equals(tableName.getCatalogName())) { + throw new SemanticException( + NOT_SUPPORTED, + statement, + "Materialized view %s created from a base table in a different catalog %s is not supported.", + viewName, tableName); + } + return toSchemaTableName(tableName); + }) + .distinct() + .collect(toImmutableList()); + + ConnectorMaterializedViewDefinition viewDefinition = new ConnectorMaterializedViewDefinition( + sql, + viewName.getSchemaName(), + viewName.getObjectName(), + baseTables, + Optional.of(session.getUser()), + analysis.getOriginalColumnMapping(statement.getQuery())); + try { + metadata.createMaterializedView(session, viewName.getCatalogName(), viewMetadata, viewDefinition, statement.isNotExists()); + } + catch (PrestoException e) { + // connectors are not required to handle the ignoreExisting flag + if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode()) || !statement.isNotExists()) { + throw e; + } + } + + return immediateFuture(null); + } + + @Override + public String explain(CreateMaterializedView statement, List parameters) + { + return "CREATE MATERIALIZED VIEW" + statement.getName(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DropColumnTask.java b/presto-main/src/main/java/com/facebook/presto/execution/DropColumnTask.java index 94f69674410c7..a3024af08690d 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/DropColumnTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/DropColumnTask.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.tree.DropColumn; @@ -57,8 +58,16 @@ public ListenableFuture execute(DropColumn statement, TransactionManager tran } return immediateFuture(null); } - TableHandle tableHandle = tableHandleOptional.get(); + Optional optionalMaterializedView = metadata.getMaterializedView(session, tableName); + if (optionalMaterializedView.isPresent()) { + if (!statement.isTableExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and drop column is not supported", tableName); + } + return immediateFuture(null); + } + + TableHandle tableHandle = tableHandleOptional.get(); String column = statement.getColumn().getValue().toLowerCase(ENGLISH); accessControl.checkCanDropColumn(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DropMaterializedViewTask.java b/presto-main/src/main/java/com/facebook/presto/execution/DropMaterializedViewTask.java new file mode 100644 index 0000000000000..d4a72d3071aa7 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/DropMaterializedViewTask.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; +import com.facebook.presto.sql.analyzer.SemanticException; +import com.facebook.presto.sql.tree.DropMaterializedView; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_MATERIALIZED_VIEW; +import static com.google.common.util.concurrent.Futures.immediateFuture; + +public class DropMaterializedViewTask + implements DataDefinitionTask +{ + @Override + public String getName() + { + return "DROP MATERIALIZED VIEW"; + } + + @Override + public ListenableFuture execute(DropMaterializedView statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) + { + Session session = stateMachine.getSession(); + QualifiedObjectName name = createQualifiedObjectName(session, statement, statement.getName()); + + Optional view = metadata.getMaterializedView(session, name); + if (!view.isPresent()) { + if (!statement.isExists()) { + throw new SemanticException(MISSING_MATERIALIZED_VIEW, statement, "Materialized view '%s' does not exist", name); + } + return immediateFuture(null); + } + + accessControl.checkCanDropTable(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), name); + accessControl.checkCanDropView(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), name); + + metadata.dropMaterializedView(session, name); + + return immediateFuture(null); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DropTableTask.java b/presto-main/src/main/java/com/facebook/presto/execution/DropTableTask.java index 46ff71efbc937..1725b9a3e94fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/DropTableTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/DropTableTask.java @@ -17,6 +17,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.tree.DropTable; @@ -29,6 +30,7 @@ import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_TABLE; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.google.common.util.concurrent.Futures.immediateFuture; public class DropTableTask @@ -54,6 +56,14 @@ public ListenableFuture execute(DropTable statement, TransactionManager trans return immediateFuture(null); } + Optional optionalMaterializedView = metadata.getMaterializedView(session, tableName); + if (optionalMaterializedView.isPresent()) { + if (!statement.isExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, not a table. Use DROP MATERIALIZED VIEW to drop.", tableName); + } + return immediateFuture(null); + } + accessControl.checkCanDropTable(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName); metadata.dropTable(session, tableHandle.get()); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RenameColumnTask.java b/presto-main/src/main/java/com/facebook/presto/execution/RenameColumnTask.java index fce90a268ffa5..b7e255aa16fab 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RenameColumnTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RenameColumnTask.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.tree.Expression; @@ -58,6 +59,15 @@ public ListenableFuture execute(RenameColumn statement, TransactionManager tr } return immediateFuture(null); } + + Optional optionalMaterializedView = metadata.getMaterializedView(session, tableName); + if (optionalMaterializedView.isPresent()) { + if (!statement.isTableExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and rename column is not supported", tableName); + } + return immediateFuture(null); + } + TableHandle tableHandle = tableHandleOptional.get(); String source = statement.getSource().getValue().toLowerCase(ENGLISH); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RenameTableTask.java b/presto-main/src/main/java/com/facebook/presto/execution/RenameTableTask.java index 003c9c38e2acb..b3f382494670c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RenameTableTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RenameTableTask.java @@ -17,6 +17,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.tree.Expression; @@ -56,6 +57,14 @@ public ListenableFuture execute(RenameTable statement, TransactionManager tra return immediateFuture(null); } + Optional optionalMaterializedView = metadata.getMaterializedView(session, tableName); + if (optionalMaterializedView.isPresent()) { + if (!statement.isExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and rename is not supported", tableName); + } + return immediateFuture(null); + } + QualifiedObjectName target = createQualifiedObjectName(session, statement, statement.getTarget()); if (!metadata.getCatalogHandle(session, target.getCatalogName()).isPresent()) { throw new SemanticException(MISSING_CATALOG, statement, "Target catalog '%s' does not exist", target.getCatalogName()); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java index 102cd6dd33f93..6f5e93d6e6b70 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.Constraint; @@ -358,6 +359,21 @@ public interface Metadata */ void dropView(Session session, QualifiedObjectName viewName); + /** + * Returns the materialized view definition for the specified materialized view name. + */ + Optional getMaterializedView(Session session, QualifiedObjectName viewName); + + /** + * Creates the specified materialized view with the specified view definition. + */ + void createMaterializedView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting); + + /** + * Drops the specified materialized view. + */ + void dropMaterializedView(Session session, QualifiedObjectName viewName); + /** * Try to locate a table index that can lookup results by indexableColumns and provide the requested outputColumns. */ diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java index c1dffee8e4ffa..d5d6a53d25e96 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorMetadataUpdateHandle; import com.facebook.presto.spi.ConnectorOutputTableHandle; import com.facebook.presto.spi.ConnectorResolvedIndex; @@ -1040,6 +1041,40 @@ public void dropView(Session session, QualifiedObjectName viewName) metadata.dropView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); } + @Override + public Optional getMaterializedView(Session session, QualifiedObjectName viewName) + { + Optional catalog = getOptionalCatalogMetadata(session, viewName.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorId connectorId = catalogMetadata.getConnectorId(session, viewName); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + + return metadata.getMaterializedView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + } + return Optional.empty(); + } + + @Override + public void createMaterializedView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + + metadata.createMaterializedView(session.toConnectorSession(connectorId), viewMetadata, viewDefinition, ignoreExisting); + } + + @Override + public void dropMaterializedView(Session session, QualifiedObjectName viewName) + { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, viewName.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + + metadata.dropMaterializedView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + } + @Override public Optional resolveIndex(Session session, TableHandle tableHandle, Set indexableColumns, Set outputColumns, TupleDomain tupleDomain) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index 5cc4f7c74931d..0d9379473b204 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -37,6 +37,7 @@ import com.facebook.presto.execution.ClusterSizeMonitor; import com.facebook.presto.execution.CommitTask; import com.facebook.presto.execution.CreateFunctionTask; +import com.facebook.presto.execution.CreateMaterializedViewTask; import com.facebook.presto.execution.CreateRoleTask; import com.facebook.presto.execution.CreateSchemaTask; import com.facebook.presto.execution.CreateTableTask; @@ -45,6 +46,7 @@ import com.facebook.presto.execution.DeallocateTask; import com.facebook.presto.execution.DropColumnTask; import com.facebook.presto.execution.DropFunctionTask; +import com.facebook.presto.execution.DropMaterializedViewTask; import com.facebook.presto.execution.DropRoleTask; import com.facebook.presto.execution.DropSchemaTask; import com.facebook.presto.execution.DropTableTask; @@ -112,6 +114,7 @@ import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateFunction; +import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; @@ -119,6 +122,7 @@ import com.facebook.presto.sql.tree.Deallocate; import com.facebook.presto.sql.tree.DropColumn; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -331,6 +335,8 @@ protected void setup(Binder binder) bindDataDefinitionTask(binder, executionBinder, DropTable.class, DropTableTask.class); bindDataDefinitionTask(binder, executionBinder, CreateView.class, CreateViewTask.class); bindDataDefinitionTask(binder, executionBinder, DropView.class, DropViewTask.class); + bindDataDefinitionTask(binder, executionBinder, CreateMaterializedView.class, CreateMaterializedViewTask.class); + bindDataDefinitionTask(binder, executionBinder, DropMaterializedView.class, DropMaterializedViewTask.class); bindDataDefinitionTask(binder, executionBinder, CreateFunction.class, CreateFunctionTask.class); bindDataDefinitionTask(binder, executionBinder, AlterFunction.class, AlterFunctionTask.class); bindDataDefinitionTask(binder, executionBinder, DropFunction.class, DropFunctionTask.class); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java index 3fc7224538a7f..6e654d01a9ee9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.security.Identity; @@ -63,9 +64,11 @@ import java.util.Set; import static com.facebook.presto.SystemSessionProperties.isCheckAccessControlOnUtilizedColumnsOnly; +import static com.facebook.presto.metadata.MetadataUtil.toSchemaTableName; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Multimaps.forMap; import static java.lang.String.format; import static java.util.Collections.emptyList; @@ -463,6 +466,11 @@ public Collection getTables() return unmodifiableCollection(tables.values()); } + public List
getTableNodes() + { + return tables.keySet().stream().map(NodeRef::getNode).collect(toImmutableList()); + } + public void registerTable(Table table, TableHandle handle) { tables.put(NodeRef.of(table), handle); @@ -709,6 +717,15 @@ public boolean isOrderByRedundant(OrderBy orderBy) return redundantOrderBy.contains(NodeRef.of(orderBy)); } + public Map> getOriginalColumnMapping(Node node) + { + return getOutputDescriptor(node).getVisibleFields().stream() + .filter(field -> field.getOriginTable().isPresent() && field.getOriginColumnName().isPresent()) + .collect(toImmutableMap( + field -> field.getName().get(), + field -> ImmutableMap.of(toSchemaTableName(field.getOriginTable().get()), field.getOriginColumnName().get()))); + } + @Immutable public static final class Insert { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java index 64b90694df629..5704f7445810e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java @@ -28,6 +28,7 @@ public enum SemanticErrorCode MISSING_CATALOG, MISSING_SCHEMA, MISSING_TABLE, + MISSING_MATERIALIZED_VIEW, MISSING_COLUMN, MISMATCHED_COLUMN_ALIASES, NOT_SUPPORTED, @@ -40,6 +41,7 @@ public enum SemanticErrorCode SCHEMA_ALREADY_EXISTS, TABLE_ALREADY_EXISTS, COLUMN_ALREADY_EXISTS, + MATERIALIZED_VIEW_ALREADY_EXISTS, DUPLICATE_RELATION, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index 0dcf4d6de91f3..8cc317fea93b9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -34,6 +34,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.TableHandle; @@ -56,6 +57,7 @@ import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateFunction; +import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; @@ -67,6 +69,7 @@ import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.DropColumn; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; import com.facebook.presto.sql.tree.DropView; @@ -197,6 +200,7 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_ORDINAL; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_PROCEDURE_ARGUMENTS; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_WINDOW_FRAME; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MATERIALIZED_VIEW_ALREADY_EXISTS; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISMATCHED_COLUMN_ALIASES; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISMATCHED_SET_COLUMN_TYPES; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_ATTRIBUTE; @@ -322,10 +326,15 @@ protected Scope visitUse(Use node, Optional scope) protected Scope visitInsert(Insert insert, Optional scope) { QualifiedObjectName targetTable = createQualifiedObjectName(session, insert, insert.getTarget()); + if (metadata.getView(session, targetTable).isPresent()) { throw new SemanticException(NOT_SUPPORTED, insert, "Inserting into views is not supported"); } + if (metadata.getMaterializedView(session, targetTable).isPresent()) { + throw new SemanticException(NOT_SUPPORTED, insert, "Inserting into materialized views is not supported"); + } + // analyze the query that creates the data Scope queryScope = process(insert.getQuery(), scope); @@ -435,10 +444,15 @@ protected Scope visitDelete(Delete node, Optional scope) { Table table = node.getTable(); QualifiedObjectName tableName = createQualifiedObjectName(session, table, table.getName()); + if (metadata.getView(session, tableName).isPresent()) { throw new SemanticException(NOT_SUPPORTED, node, "Deleting from views is not supported"); } + if (metadata.getMaterializedView(session, tableName).isPresent()) { + throw new SemanticException(NOT_SUPPORTED, node, "Deleting from materialized views is not supported"); + } + // Analyzer checks for select permissions but DELETE has a separate permission, so disable access checks // TODO: we shouldn't need to create a new analyzer. The access control should be carried in the context object StatementAnalyzer analyzer = new StatementAnalyzer( @@ -569,6 +583,40 @@ protected Scope visitCreateView(CreateView node, Optional scope) return createAndAssignScope(node, scope); } + @Override + protected Scope visitCreateMaterializedView(CreateMaterializedView node, Optional scope) + { + analysis.setUpdateType("CREATE MATERIALIZED VIEW"); + + QualifiedObjectName viewName = createQualifiedObjectName(session, node, node.getName()); + analysis.setCreateTableDestination(viewName); + + Optional viewHandle = metadata.getTableHandle(session, viewName); + if (viewHandle.isPresent()) { + if (node.isNotExists()) { + return createAndAssignScope(node, scope); + } + throw new SemanticException(MATERIALIZED_VIEW_ALREADY_EXISTS, node, "Destination materialized view '%s' already exists", viewName); + } + + validateProperties(node.getProperties(), scope); + + analysis.setCreateTableProperties(mapFromProperties(node.getProperties())); + analysis.setCreateTableComment(node.getComment()); + + accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), viewName); + accessControl.checkCanCreateView(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), viewName); + + // analyze the query that creates the table + Scope queryScope = process(node.getQuery(), scope); + + validateColumns(node, queryScope.getRelationType()); + + validateBaseTables(analysis.getTableNodes(), node); + + return createAndAssignScope(node, scope); + } + @Override protected Scope visitCreateFunction(CreateFunction node, Optional scope) { @@ -717,6 +765,12 @@ protected Scope visitDropView(DropView node, Optional scope) return createAndAssignScope(node, scope); } + @Override + protected Scope visitDropMaterializedView(DropMaterializedView node, Optional scope) + { + return createAndAssignScope(node, scope); + } + @Override protected Scope visitStartTransaction(StartTransaction node, Optional scope) { @@ -822,6 +876,22 @@ private void validateColumnAliases(List columnAliases, int sourceCol } } + private void validateBaseTables(List
baseTables, Node node) + { + for (Table baseTable : baseTables) { + QualifiedObjectName baseName = createQualifiedObjectName(session, baseTable, baseTable.getName()); + + Optional optionalMaterializedView = metadata.getMaterializedView(session, baseName); + if (optionalMaterializedView.isPresent()) { + throw new SemanticException( + NOT_SUPPORTED, + baseTable, + "%s on a materialized view %s is not supported.", + node.getClass().getSimpleName(), baseName); + } + } + } + @Override protected Scope visitExplain(Explain node, Optional scope) throws SemanticException @@ -1011,6 +1081,15 @@ protected Scope visitTable(Table table, Optional scope) return createAndAssignScope(table, scope, outputFields); } + Optional optionalMaterializedView = metadata.getMaterializedView(session, name); + if (optionalMaterializedView.isPresent()) { + ConnectorMaterializedViewDefinition view = optionalMaterializedView.get(); + Query query = parseView(view.getOriginalSql(), name, table); + + // TODO: stitch with base table data that is not updated to the materialized view yet, if any, + // so that querying materialized view returns consistent result with querying the base tables. + } + Optional tableHandle = metadata.getTableHandle(session, name); if (!tableHandle.isPresent()) { if (!metadata.getCatalogHandle(session, name.getCatalogName()).isPresent()) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java index 25a72611b9056..709d85f068c9b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java @@ -22,6 +22,7 @@ import com.facebook.presto.metadata.ViewDefinition; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -441,9 +442,13 @@ protected Node visitShowCreate(ShowCreate node, Void context) { QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName()); Optional viewDefinition = metadata.getView(session, objectName); + Optional materializedViewDefinition = metadata.getMaterializedView(session, objectName); if (node.getType() == VIEW) { if (!viewDefinition.isPresent()) { + if (materializedViewDefinition.isPresent()) { + throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a materialized view, not a view", objectName); + } if (metadata.getTableHandle(session, objectName).isPresent()) { throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a view", objectName); } @@ -459,6 +464,9 @@ protected Node visitShowCreate(ShowCreate node, Void context) if (viewDefinition.isPresent()) { throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a view, not a table", objectName); } + if (materializedViewDefinition.isPresent()) { + throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a materialized view, not a table", objectName); + } Optional tableHandle = metadata.getTableHandle(session, objectName); if (!tableHandle.isPresent()) { diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 015b8a073f8c7..8bfa5118fd8d8 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -45,11 +45,13 @@ import com.facebook.presto.execution.AlterFunctionTask; import com.facebook.presto.execution.CommitTask; import com.facebook.presto.execution.CreateFunctionTask; +import com.facebook.presto.execution.CreateMaterializedViewTask; import com.facebook.presto.execution.CreateTableTask; import com.facebook.presto.execution.CreateViewTask; import com.facebook.presto.execution.DataDefinitionTask; import com.facebook.presto.execution.DeallocateTask; import com.facebook.presto.execution.DropFunctionTask; +import com.facebook.presto.execution.DropMaterializedViewTask; import com.facebook.presto.execution.DropTableTask; import com.facebook.presto.execution.DropViewTask; import com.facebook.presto.execution.Lifespan; @@ -165,10 +167,12 @@ import com.facebook.presto.sql.tree.AlterFunction; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateFunction; +import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Deallocate; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropTable; import com.facebook.presto.sql.tree.DropView; import com.facebook.presto.sql.tree.Explain; @@ -461,11 +465,13 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, dataDefinitionTask = ImmutableMap., DataDefinitionTask>builder() .put(CreateTable.class, new CreateTableTask()) .put(CreateView.class, new CreateViewTask(jsonCodec(ViewDefinition.class), sqlParser, new FeaturesConfig())) + .put(CreateMaterializedView.class, new CreateMaterializedViewTask(sqlParser)) .put(CreateFunction.class, new CreateFunctionTask(sqlParser)) .put(AlterFunction.class, new AlterFunctionTask(sqlParser)) .put(DropFunction.class, new DropFunctionTask(sqlParser)) .put(DropTable.class, new DropTableTask()) .put(DropView.class, new DropViewTask()) + .put(DropMaterializedView.class, new DropMaterializedViewTask()) .put(RenameColumn.class, new RenameColumnTask()) .put(RenameTable.class, new RenameTableTask()) .put(ResetSession.class, new ResetSessionTask()) diff --git a/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java b/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java index f6a8e779d65e2..199d3bd3a8be2 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java @@ -20,6 +20,7 @@ import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.CreateFunction; +import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; @@ -31,6 +32,7 @@ import com.facebook.presto.sql.tree.DescribeOutput; import com.facebook.presto.sql.tree.DropColumn; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -114,6 +116,8 @@ private StatementUtils() {} builder.put(DropTable.class, QueryType.DATA_DEFINITION); builder.put(CreateView.class, QueryType.DATA_DEFINITION); builder.put(DropView.class, QueryType.DATA_DEFINITION); + builder.put(CreateMaterializedView.class, QueryType.DATA_DEFINITION); + builder.put(DropMaterializedView.class, QueryType.DATA_DEFINITION); builder.put(CreateFunction.class, QueryType.DATA_DEFINITION); builder.put(AlterFunction.class, QueryType.DATA_DEFINITION); builder.put(DropFunction.class, QueryType.DATA_DEFINITION); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java new file mode 100644 index 0000000000000..737fe0157c6dd --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java @@ -0,0 +1,292 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.metadata.AbstractMockMetadata; +import com.facebook.presto.metadata.Catalog; +import com.facebook.presto.metadata.CatalogManager; +import com.facebook.presto.metadata.ColumnPropertyManager; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.metadata.TableMetadata; +import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.metadata.ViewDefinition; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.security.AllowAllAccessControl; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.resourceGroups.ResourceGroupId; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.CreateMaterializedView; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; + +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; +import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; +import static com.facebook.presto.spi.session.PropertyMetadata.stringProperty; +import static com.facebook.presto.testing.TestingSession.createBogusTestingCatalog; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +@Test(singleThreaded = true) +public class TestCreateMaterializedViewTask +{ + private static final String CATALOG_NAME = "catalog"; + private static final String SCHEMA_NAME = "schema"; + private static final String MATERIALIZED_VIEW_A = "materialized_view_a"; + private static final String MATERIALIZED_VIEW_B = "materialized_view_b"; + private static final String TABLE_A = "table_a"; + + private TransactionManager transactionManager; + private Session testSession; + + private AccessControl accessControl; + + private ExecutorService executorService; + + private MockMetadata metadata; + + @BeforeMethod + public void setUp() + { + CatalogManager catalogManager = new CatalogManager(); + Catalog testCatalog = createBogusTestingCatalog(CATALOG_NAME); + catalogManager.registerCatalog(testCatalog); + + TablePropertyManager tablePropertyManager = new TablePropertyManager(); + tablePropertyManager.addProperties(testCatalog.getConnectorId(), + ImmutableList.of(stringProperty("baz", "test property", null, false))); + + ColumnPropertyManager columnPropertyManager = new ColumnPropertyManager(); + columnPropertyManager.addProperties(testCatalog.getConnectorId(), ImmutableList.of()); + + FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager(); + + transactionManager = createTestTransactionManager(catalogManager); + testSession = testSessionBuilder() + .setTransactionId(transactionManager.beginTransaction(false)) + .build(); + + accessControl = new AllowAllAccessControl(); + + executorService = newCachedThreadPool(daemonThreadsNamed("test-%s")); + + metadata = new MockMetadata( + functionAndTypeManager, + tablePropertyManager, + columnPropertyManager, + testCatalog.getConnectorId()); + } + + @Test + public void testCreateMaterializedViewNotExistsTrue() + { + SqlParser parser = new SqlParser(); + String sql = String.format("CREATE MATERIALIZED VIEW IF NOT EXISTS %s AS SELECT 2021 AS col_0 FROM %s", MATERIALIZED_VIEW_A, TABLE_A); + CreateMaterializedView statement = (CreateMaterializedView) parser.createStatement(sql, ParsingOptions.builder().build()); + + QueryStateMachine stateMachine = QueryStateMachine.begin( + sql, + testSession, + URI.create("fake://uri"), + new ResourceGroupId("test"), + Optional.empty(), + false, + transactionManager, + accessControl, + executorService, + metadata, + WarningCollector.NOOP); + + getFutureValue(new CreateMaterializedViewTask(parser).execute(statement, transactionManager, metadata, accessControl, stateMachine, emptyList())); + + assertEquals(metadata.getCreateMaterializedViewCallCount(), 1); + } + + @Test + public void testCreateMaterializedViewExistsFalse() + { + SqlParser parser = new SqlParser(); + String sql = String.format("CREATE MATERIALIZED VIEW %s AS SELECT 2021 AS col_0 FROM %s", MATERIALIZED_VIEW_B, TABLE_A); + CreateMaterializedView statement = (CreateMaterializedView) parser.createStatement(sql, ParsingOptions.builder().build()); + + QueryStateMachine stateMachine = QueryStateMachine.begin( + sql, + testSession, + URI.create("fake://uri"), + new ResourceGroupId("test"), + Optional.empty(), + false, + transactionManager, + accessControl, + executorService, + metadata, + WarningCollector.NOOP); + + try { + getFutureValue(new CreateMaterializedViewTask(parser).execute(statement, transactionManager, metadata, accessControl, stateMachine, emptyList())); + fail("expected exception"); + } + catch (RuntimeException e) { + // Expected + assertTrue(e instanceof PrestoException); + PrestoException prestoException = (PrestoException) e; + assertEquals(prestoException.getErrorCode(), ALREADY_EXISTS.toErrorCode()); + } + + assertEquals(metadata.getCreateMaterializedViewCallCount(), 0); + } + + private static class MockMetadata + extends AbstractMockMetadata + { + private final FunctionAndTypeManager functionAndTypeManager; + private final TablePropertyManager tablePropertyManager; + private final ColumnPropertyManager columnPropertyManager; + private final ConnectorId catalogHandle; + + private final List materializedViews = new CopyOnWriteArrayList<>(); + + public MockMetadata( + FunctionAndTypeManager functionAndTypeManager, + TablePropertyManager tablePropertyManager, + ColumnPropertyManager columnPropertyManager, + ConnectorId catalogHandle) + { + this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + this.tablePropertyManager = requireNonNull(tablePropertyManager, "tablePropertyManager is null"); + this.columnPropertyManager = requireNonNull(columnPropertyManager, "columnPropertyManager is null"); + this.catalogHandle = requireNonNull(catalogHandle, "catalogHandle is null"); + } + + @Override + public void createMaterializedView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + if (!ignoreExisting) { + throw new PrestoException(ALREADY_EXISTS, "Materialized view already exists"); + } + this.materializedViews.add(viewMetadata); + } + + public int getCreateMaterializedViewCallCount() + { + return materializedViews.size(); + } + + @Override + public TablePropertyManager getTablePropertyManager() + { + return tablePropertyManager; + } + + @Override + public ColumnPropertyManager getColumnPropertyManager() + { + return columnPropertyManager; + } + + @Override + public FunctionAndTypeManager getFunctionAndTypeManager() + { + return functionAndTypeManager; + } + + @Override + public Type getType(TypeSignature signature) + { + return functionAndTypeManager.getType(signature); + } + + @Override + public Optional getTableHandle(Session session, QualifiedObjectName tableName) + { + if (tableName.getObjectName().equals(MATERIALIZED_VIEW_A)) { + return Optional.empty(); + } + if (tableName.getObjectName().equals(TABLE_A)) { + return Optional.of(new TableHandle( + catalogHandle, + new ConnectorTableHandle() {}, + new ConnectorTransactionHandle() {}, + Optional.empty())); + } + return Optional.empty(); + } + + @Override + public Map getColumnHandles(Session session, TableHandle tableHandle) + { + return emptyMap(); + } + + @Override + public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) + { + return new TableMetadata( + catalogHandle, + new ConnectorTableMetadata(new SchemaTableName(SCHEMA_NAME, TABLE_A), emptyList())); + } + + @Override + public Optional getView(Session session, QualifiedObjectName viewName) + { + return Optional.empty(); + } + + @Override + public Optional getMaterializedView(Session session, QualifiedObjectName viewName) + { + return Optional.empty(); + } + + @Override + public Optional getCatalogHandle(Session session, String catalogName) + { + if (catalogHandle.getCatalogName().equals(catalogName)) { + return Optional.of(catalogHandle); + } + return Optional.empty(); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index 8a03cbd91ccd7..98d33d6f6f4e0 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.QueryId; @@ -412,6 +413,24 @@ public void dropView(Session session, QualifiedObjectName viewName) throw new UnsupportedOperationException(); } + @Override + public Optional getMaterializedView(Session session, QualifiedObjectName viewName) + { + throw new UnsupportedOperationException(); + } + + @Override + public void createMaterializedView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + throw new UnsupportedOperationException(); + } + + @Override + public void dropMaterializedView(Session session, QualifiedObjectName viewName) + { + throw new UnsupportedOperationException(); + } + @Override public Optional resolveIndex(Session session, TableHandle tableHandle, Set indexableColumns, Set outputColumns, TupleDomain tupleDomain) { diff --git a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 index b7598502ef57e..82d74c9d53147 100644 --- a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 +++ b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 @@ -64,6 +64,7 @@ statement | CREATE MATERIALIZED VIEW (IF NOT EXISTS)? qualifiedName (COMMENT string)? (WITH properties)? AS (query | '('query')') #createMaterializedView + | DROP MATERIALIZED VIEW (IF EXISTS)? qualifiedName #dropMaterializedView | CREATE (OR REPLACE)? TEMPORARY? FUNCTION functionName=qualifiedName '(' (sqlParameterDeclaration (',' sqlParameterDeclaration)*)? ')' RETURNS returnType=type diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java index 3258fb36f29ae..8f15bc51ff033 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java @@ -37,6 +37,7 @@ import com.facebook.presto.sql.tree.DescribeOutput; import com.facebook.presto.sql.tree.DropColumn; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -669,6 +670,18 @@ protected Void visitDropView(DropView node, Integer context) return null; } + @Override + protected Void visitDropMaterializedView(DropMaterializedView node, Integer context) + { + builder.append("DROP MATERIALIZED VIEW "); + if (node.isExists()) { + builder.append("IF EXISTS "); + } + builder.append(node.getName()); + + return null; + } + @Override protected Void visitExplain(Explain node, Integer indent) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java index a65b30b0a3b7b..9919ab5f6ee50 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java @@ -54,6 +54,7 @@ import com.facebook.presto.sql.tree.DoubleLiteral; import com.facebook.presto.sql.tree.DropColumn; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -350,6 +351,12 @@ public Node visitDropView(SqlBaseParser.DropViewContext context) return new DropView(getLocation(context), getQualifiedName(context.qualifiedName()), context.EXISTS() != null); } + @Override + public Node visitDropMaterializedView(SqlBaseParser.DropMaterializedViewContext context) + { + return new DropMaterializedView(Optional.of(getLocation(context)), getQualifiedName(context.qualifiedName()), context.EXISTS() != null); + } + @Override public Node visitInsertInto(SqlBaseParser.InsertIntoContext context) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java index 882219d718e0c..af9564137b7d0 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java @@ -597,6 +597,11 @@ protected R visitCreateMaterializedView(CreateMaterializedView node, C context) return visitStatement(node, context); } + protected R visitDropMaterializedView(DropMaterializedView node, C context) + { + return visitStatement(node, context); + } + protected R visitCreateFunction(CreateFunction node, C context) { return visitStatement(node, context); diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java index 87c25bb3a833c..2200a39f10e30 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java @@ -561,6 +561,17 @@ protected R visitCreateView(CreateView node, C context) return null; } + @Override + protected R visitCreateMaterializedView(CreateMaterializedView node, C context) + { + process(node.getQuery(), context); + for (Property property : node.getProperties()) { + process(property, context); + } + + return null; + } + @Override protected R visitSetSession(SetSession node, C context) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/DropMaterializedView.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DropMaterializedView.java new file mode 100644 index 0000000000000..e0088bbc35cef --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DropMaterializedView.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class DropMaterializedView + extends Statement +{ + private final QualifiedName name; + private final boolean exists; + + public DropMaterializedView(Optional location, QualifiedName name, boolean exists) + { + super(location); + this.name = requireNonNull(name); + this.exists = exists; + } + + public QualifiedName getName() + { + return name; + } + + public boolean isExists() + { + return exists; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitDropMaterializedView(this, context); + } + + @Override + public List getChildren() + { + return ImmutableList.of(); + } + + @Override + public int hashCode() + { + return Objects.hash(name, exists); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + DropMaterializedView o = (DropMaterializedView) obj; + return Objects.equals(name, o.name) + && (exists == o.exists); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("name", name) + .add("exists", exists) + .toString(); + } +} diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java index cf7c3bd69758f..7ffef1dec2620 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java @@ -51,6 +51,7 @@ import com.facebook.presto.sql.tree.DoubleLiteral; import com.facebook.presto.sql.tree.DropColumn; import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.DropMaterializedView; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -1364,6 +1365,18 @@ public void testDropView() assertStatement("DROP VIEW IF EXISTS a.b.c", new DropView(QualifiedName.of("a", "b", "c"), true)); } + @Test + public void testDropMaterializedView() + { + assertStatement("DROP MATERIALIZED VIEW a", new DropMaterializedView(Optional.empty(), QualifiedName.of("a"), false)); + assertStatement("DROP MATERIALIZED VIEW a.b", new DropMaterializedView(Optional.empty(), QualifiedName.of("a", "b"), false)); + assertStatement("DROP MATERIALIZED VIEW a.b.c", new DropMaterializedView(Optional.empty(), QualifiedName.of("a", "b", "c"), false)); + + assertStatement("DROP MATERIALIZED VIEW IF EXISTS a", new DropMaterializedView(Optional.empty(), QualifiedName.of("a"), true)); + assertStatement("DROP MATERIALIZED VIEW IF EXISTS a.b", new DropMaterializedView(Optional.empty(), QualifiedName.of("a", "b"), true)); + assertStatement("DROP MATERIALIZED VIEW IF EXISTS a.b.c", new DropMaterializedView(Optional.empty(), QualifiedName.of("a", "b", "c"), true)); + } + @Test public void testDropFunction() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorMaterializedViewDefinition.java b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorMaterializedViewDefinition.java new file mode 100644 index 0000000000000..55773a8cc7c25 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorMaterializedViewDefinition.java @@ -0,0 +1,236 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import static java.util.Collections.unmodifiableList; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; + +public final class ConnectorMaterializedViewDefinition +{ + private final String originalSql; + private final String schema; + private final String table; + private final List baseTables; + private final Optional owner; + private final List columnMappings; + + @JsonCreator + public ConnectorMaterializedViewDefinition( + @JsonProperty("originalSql") String originalSql, + @JsonProperty("schema") String schema, + @JsonProperty("table") String table, + @JsonProperty("baseTables") List baseTables, + @JsonProperty("owner") Optional owner, + @JsonProperty("columnMapping") List columnMappings) + { + this.originalSql = requireNonNull(originalSql, "originalSql is null"); + this.schema = requireNonNull(schema, "schema is null"); + this.table = requireNonNull(table, "table is null"); + this.baseTables = unmodifiableList(new ArrayList<>(requireNonNull(baseTables, "baseTables is null"))); + this.owner = requireNonNull(owner, "owner is null"); + this.columnMappings = unmodifiableList(new ArrayList<>(requireNonNull(columnMappings, "columnMappings is null"))); + } + + @JsonIgnore + public ConnectorMaterializedViewDefinition( + String originalSql, + String schema, + String table, + List baseTables, + Optional owner, + Map> columnMappingsAsMap) + { + this(originalSql, schema, table, baseTables, owner, convertFromMapToColumnMappings(requireNonNull(columnMappingsAsMap, "columnMappings is null"), new SchemaTableName(schema, table))); + } + + @JsonProperty + public String getOriginalSql() + { + return originalSql; + } + + @JsonProperty + public String getSchema() + { + return schema; + } + + @JsonProperty + public String getTable() + { + return table; + } + + @JsonProperty + public List getBaseTables() + { + return baseTables; + } + + @JsonProperty + public Optional getOwner() + { + return owner; + } + + @JsonProperty + public List getColumnMappings() + { + return columnMappings; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder("ConnectorMaterializedViewDefinition{"); + sb.append("originalSql=").append(originalSql); + sb.append(",schema=").append(schema); + sb.append(",table=").append(table); + sb.append(",baseTables=").append(baseTables); + sb.append(",owner=").append(owner.orElse(null)); + sb.append(",columnMappings=").append(columnMappings); + sb.append("}"); + return sb.toString(); + } + + @JsonIgnore + public Map> getColumnMappingsAsMap() + { + return columnMappings.stream() + .collect(toMap( + mapping -> mapping.getViewColumn().getColumnName(), + mapping -> mapping.getBaseTableColumns().stream().collect(toMap(TableColumn::getTableName, TableColumn::getColumnName)))); + } + + @JsonIgnore + private static List convertFromMapToColumnMappings(Map> columnMappings, SchemaTableName sourceTable) + { + List columnMappingList = new ArrayList<>(); + + for (String sourceColumn : columnMappings.keySet()) { + TableColumn viewColumn = new TableColumn(sourceTable, sourceColumn); + + List baseTableColumns = new ArrayList<>(); + for (SchemaTableName baseTable : columnMappings.get(sourceColumn).keySet()) { + baseTableColumns.add(new TableColumn(baseTable, columnMappings.get(sourceColumn).get(baseTable))); + } + + columnMappingList.add(new ColumnMapping(viewColumn, unmodifiableList(baseTableColumns))); + } + + return unmodifiableList(columnMappingList); + } + + public static final class ColumnMapping + { + private final TableColumn viewColumn; + private final List baseTableColumns; + + @JsonCreator + public ColumnMapping( + @JsonProperty("viewColumn") TableColumn viewColumn, + @JsonProperty("baseTableColumns") List baseTableColumns) + { + this.viewColumn = requireNonNull(viewColumn, "viewColumn is null"); + this.baseTableColumns = unmodifiableList(new ArrayList<>(requireNonNull(baseTableColumns, "baseTableColumns is null"))); + } + + @JsonProperty + public TableColumn getViewColumn() + { + return viewColumn; + } + + @JsonProperty + public List getBaseTableColumns() + { + return baseTableColumns; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder("ColumnMapping{"); + sb.append("viewColumn=").append(viewColumn); + sb.append(",baseTableColumns=").append(baseTableColumns); + sb.append("}"); + return sb.toString(); + } + } + + public static final class TableColumn + { + private final SchemaTableName tableName; + private final String columnName; + + @JsonCreator + public TableColumn( + @JsonProperty("tableName") SchemaTableName tableName, + @JsonProperty("columnName") String columnName) + { + this.tableName = requireNonNull(tableName, "tableName is null"); + this.columnName = requireNonNull(columnName, "columnName is null"); + } + + @JsonProperty + public SchemaTableName getTableName() + { + return tableName; + } + + @JsonProperty + public String getColumnName() + { + return columnName; + } + + @Override + public int hashCode() + { + return Objects.hash(tableName, columnName); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TableColumn that = (TableColumn) o; + return Objects.equals(this.columnName, that.columnName) && + Objects.equals(this.tableName, that.tableName); + } + + @Override + public String toString() + { + return tableName + ":" + columnName; + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewNotFoundException.java b/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewNotFoundException.java new file mode 100644 index 0000000000000..baf03e2fa21cd --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewNotFoundException.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class MaterializedViewNotFoundException + extends NotFoundException +{ + private final SchemaTableName viewName; + + public MaterializedViewNotFoundException(SchemaTableName viewName) + { + this(viewName, format("Materialized view '%s' not found", viewName)); + } + + public MaterializedViewNotFoundException(SchemaTableName viewName, String message) + { + super(message); + this.viewName = requireNonNull(viewName, "viewName is null"); + } + + public MaterializedViewNotFoundException(SchemaTableName viewName, Throwable cause) + { + this(viewName, format("Materialzied view '%s' not found", viewName), cause); + } + + public MaterializedViewNotFoundException(SchemaTableName viewName, String message, Throwable cause) + { + super(message, cause); + this.viewName = requireNonNull(viewName, "viewName is null"); + } + + public SchemaTableName getViewName() + { + return viewName; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java index dba796f5c63a1..233dbd7080808 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorMetadataUpdateHandle; import com.facebook.presto.spi.ConnectorNewTableLayout; import com.facebook.presto.spi.ConnectorOutputTableHandle; @@ -560,6 +561,30 @@ default Map getViews(ConnectorSession return emptyMap(); } + /** + * Gets the materialized view data for the specified materialized view name. + */ + default Optional getMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + return Optional.empty(); + } + + /** + * Create the specified materialized view. The data for the materialized view is opaque to the connector. + */ + default void createMaterializedView(ConnectorSession session, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + throw new PrestoException(NOT_SUPPORTED, "This connector does not support creating materialized views"); + } + + /** + * Drop the specified materialized view. + */ + default void dropMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + throw new PrestoException(NOT_SUPPORTED, "This connector does not support dropping materialized views"); + } + /** * @return whether delete without table scan is supported */ diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java index 696a3159fccdd..ab9678f5a809d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; import com.facebook.presto.spi.ConnectorMetadataUpdateHandle; import com.facebook.presto.spi.ConnectorNewTableLayout; import com.facebook.presto.spi.ConnectorOutputTableHandle; @@ -475,6 +476,30 @@ public Map getViews(ConnectorSession s } } + @Override + public Optional getMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.getMaterializedView(session, viewName); + } + } + + @Override + public void createMaterializedView(ConnectorSession session, ConnectorTableMetadata viewMetadata, ConnectorMaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + delegate.createMaterializedView(session, viewMetadata, viewDefinition, ignoreExisting); + } + } + + @Override + public void dropMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + delegate.dropMaterializedView(session, viewName); + } + } + @Override public ColumnHandle getUpdateRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle) {