Support materialized view creation, drop and query

presto-hive-metastore/src/main/java/com/facebook/presto/hive/MaterializedViewAlreadyExistsException.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.hive;
+
+import com.facebook.presto.spi.PrestoException;
+import com.facebook.presto.spi.SchemaTableName;
+
+import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS;
+import static java.lang.String.format;
+
+public class MaterializedViewAlreadyExistsException
+        extends PrestoException
+{
+    private final SchemaTableName viewName;
+
+    public MaterializedViewAlreadyExistsException(SchemaTableName viewName)
+    {
+        this(viewName, format("Materialized view already exists: '%s'", viewName));
+    }
+
+    public MaterializedViewAlreadyExistsException(SchemaTableName viewName, String message)
+    {
+        this(viewName, message, null);
+    }
+
+    public MaterializedViewAlreadyExistsException(SchemaTableName viewName, String message, Throwable cause)
+    {
+        super(ALREADY_EXISTS, message, cause);
+        this.viewName = viewName;
+    }
+
+    public SchemaTableName getViewName()
+    {
+        return viewName;
+    }
+}

presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java

@@ -48,6 +48,7 @@
 import com.facebook.presto.spi.statistics.ColumnStatisticType;
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -102,6 +103,7 @@
 import static com.facebook.presto.spi.statistics.ColumnStatisticType.NUMBER_OF_TRUE_VALUES;
 import static com.facebook.presto.spi.statistics.ColumnStatisticType.TOTAL_SIZE_IN_BYTES;
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.base.Strings.padEnd;
 import static com.google.common.io.BaseEncoding.base16;
@@ -131,6 +133,7 @@ public class MetastoreUtil
     public static final String PRESTO_OFFLINE = "presto_offline";
     public static final String AVRO_SCHEMA_URL_KEY = "avro.schema.url";
     public static final String PRESTO_VIEW_FLAG = "presto_view";
+    public static final String PRESTO_MATERIALIZED_VIEW_FLAG = "presto_materialized_view";
     public static final String PRESTO_QUERY_ID_NAME = "presto_query_id";
     public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__";
     @SuppressWarnings("OctalInteger")
@@ -654,6 +657,17 @@ public static boolean isPrestoView(Table table)
         return "true".equals(table.getParameters().get(PRESTO_VIEW_FLAG));
     }
 
+    public static boolean isPrestoMaterializedView(Table table)
+    {
+        if ("true".equals(table.getParameters().get(PRESTO_MATERIALIZED_VIEW_FLAG))) {
+            checkState(
+                    !table.getViewOriginalText().map(Strings::isNullOrEmpty).orElse(true),
+                    "viewOriginalText field is not set for the Table metadata of materialized view %s.", table.getTableName());
+            return true;
+        }
+        return false;
+    }
+
     private static String getRenameErrorMessage(Path source, Path target)
     {
         return format("Error moving data files from %s to final location %s", source, target);

presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java

@@ -20,6 +20,7 @@ public enum PrestoTableType
     MANAGED_TABLE,
     EXTERNAL_TABLE,
     VIRTUAL_VIEW,
+    MATERIALIZED_VIEW,
     TEMPORARY_TABLE,
     OTHER, // Some Hive implementations define additional table types
     /**/;

presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java

@@ -77,6 +77,7 @@
 import static com.facebook.presto.hive.metastore.MetastoreUtil.renameFile;
 import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues;
 import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE;
+import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW;
 import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE;
 import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW;
 import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.SUBTRACT;
@@ -491,7 +492,7 @@ public synchronized void truncateUnpartitionedTable(ConnectorSession session, St
         if (!table.isPresent()) {
             throw new TableNotFoundException(schemaTableName);
         }
-        if (!table.get().getTableType().equals(MANAGED_TABLE)) {
+        if (!table.get().getTableType().equals(MANAGED_TABLE) && !table.get().getTableType().equals(MATERIALIZED_VIEW)) {
             throw new PrestoException(NOT_SUPPORTED, "Cannot delete from non-managed Hive table");
         }
         if (!table.get().getPartitionColumns().isEmpty()) {
@@ -1132,7 +1133,7 @@ private void prepareAddTable(HdfsContext context, TableAndMore tableAndMore)
                 return;
             }
 
-            if (table.getTableType().equals(MANAGED_TABLE)) {
+            if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) {
                 String targetLocation = table.getStorage().getLocation();
                 checkArgument(!targetLocation.isEmpty(), "target location is empty");
                 Optional<Path> currentPath = tableAndMore.getCurrentLocation();

presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java

@@ -87,6 +87,7 @@
 import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn;
 import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE;
 import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE;
+import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW;
 import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE;
 import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW;
 import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS;
@@ -235,7 +236,7 @@ public synchronized void createTable(Table table, PrincipalPrivileges principalP
         if (table.getTableType().equals(VIRTUAL_VIEW)) {
             checkArgument(table.getStorage().getLocation().isEmpty(), "Storage location for view must be empty");
         }
-        else if (table.getTableType().equals(MANAGED_TABLE)) {
+        else if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) {
             if (!tableMetadataDirectory.equals(new Path(table.getStorage().getLocation()))) {
                 throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory must be " + tableMetadataDirectory);
             }
@@ -412,7 +413,7 @@ public synchronized void dropTable(String databaseName, String tableName, boolea
         Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName);
 
         // It is safe to delete the whole meta directory for external tables and views
-        if (!table.getTableType().equals(MANAGED_TABLE) || deleteData) {
+        if ((!table.getTableType().equals(MANAGED_TABLE) && !table.getTableType().equals(MATERIALIZED_VIEW)) || deleteData) {
             deleteMetadataDirectory(tableMetadataDirectory);
         }
         else {
@@ -566,7 +567,7 @@ public synchronized void addPartitions(String databaseName, String tableName, Li
 
         Table table = getRequiredTable(databaseName, tableName);
 
-        checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(table.getTableType()), "Invalid table type: %s", table.getTableType());
+        checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, MATERIALIZED_VIEW).contains(table.getTableType()), "Invalid table type: %s", table.getTableType());
 
         try {
             Map<Path, byte[]> schemaFiles = new LinkedHashMap<>();
@@ -614,7 +615,7 @@ private void verifiedPartition(Table table, Partition partition)
     {
         Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
 
-        if (table.getTableType().equals(MANAGED_TABLE)) {
+        if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) {
             if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) {
                 throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory);
             }

presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java

@@ -40,6 +40,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.primitives.Shorts;
+import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -99,14 +100,17 @@
 import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT;
 import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE;
 import static com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY;
+import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_MATERIALIZED_VIEW_FLAG;
 import static com.facebook.presto.hive.metastore.MetastoreUtil.fromMetastoreDistinctValuesCount;
 import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE;
 import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE;
+import static com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW;
 import static com.facebook.presto.hive.metastore.PrestoTableType.OTHER;
 import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW;
 import static com.facebook.presto.spi.security.PrincipalType.ROLE;
 import static com.facebook.presto.spi.security.PrincipalType.USER;
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.emptyToNull;
 import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
@@ -146,11 +150,20 @@ public static org.apache.hadoop.hive.metastore.api.Database toMetastoreApiDataba
     public static org.apache.hadoop.hive.metastore.api.Table toMetastoreApiTable(Table table, PrincipalPrivileges privileges)
     {
         org.apache.hadoop.hive.metastore.api.Table result = new org.apache.hadoop.hive.metastore.api.Table();
+
         result.setDbName(table.getDatabaseName());
         result.setTableName(table.getTableName());
         result.setOwner(table.getOwner());
-        checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW).contains(table.getTableType()), "Invalid table type: %s", table.getTableType());
-        result.setTableType(table.getTableType().name());
+
+        PrestoTableType tableType = table.getTableType();
+        checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, MATERIALIZED_VIEW).contains(tableType), "Invalid table type: %s", table.getTableType());
+        // TODO: remove the table type change after Hive 3.0 upgrade.
+        // TableType.MATERIALIZED_VIEW is not supported by Hive metastore until Hive 3.0. Use MANAGED_TABLE for now.
+        if (MATERIALIZED_VIEW.equals(tableType)) {
+            tableType = MANAGED_TABLE;
+        }
+        result.setTableType(tableType.name());
+
         result.setParameters(table.getParameters());
         result.setPartitionKeys(table.getPartitionColumns().stream().map(ThriftMetastoreUtil::toMetastoreApiFieldSchema).collect(toList()));
         result.setSd(makeStorageDescriptor(table.getTableName(), table.getDataColumns(), table.getStorage()));
@@ -400,11 +413,19 @@ public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.T
             throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor");
         }
 
+        // TODO: remove the table type change after Hive 3.0 update.
+        // Materialized view fetched from Hive Metastore uses TableType.MANAGED_TABLE before Hive 3.0. Cast it back to MATERIALIZED_VIEW here.
+        PrestoTableType tableType = PrestoTableType.optionalValueOf(table.getTableType()).orElse(OTHER);
+        if (table.getParameters() != null && "true".equals(table.getParameters().get(PRESTO_MATERIALIZED_VIEW_FLAG))) {
+            checkState(TableType.MANAGED_TABLE.name().equals(table.getTableType()), "Materialized view %s has incorrect table type %s from Metastore.", table.getTableName(), table.getTableType());
+            tableType = MATERIALIZED_VIEW;
+        }
+
         Table.Builder tableBuilder = Table.builder()
                 .setDatabaseName(table.getDbName())
                 .setTableName(table.getTableName())
                 .setOwner(nullToEmpty(table.getOwner()))
-                .setTableType(PrestoTableType.optionalValueOf(table.getTableType()).orElse(OTHER))
+                .setTableType(tableType)
                 .setDataColumns(schema.stream()
                         .map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema)
                         .collect(toList()))